Merge ~racb/ubuntu/+source/memcached:upstream-bump-1.5.6 into ubuntu/+source/memcached:ubuntu/devel
- Git
- lp:~racb/ubuntu/+source/memcached
- upstream-bump-1.5.6
- Merge into ubuntu/devel
Status: | Merged | ||||||||
---|---|---|---|---|---|---|---|---|---|
Merge reported by: | Robie Basak | ||||||||
Merged at revision: | 6f2e133e12e997cbdf6d482a8bfa0d35c4932b62 | ||||||||
Proposed branch: | ~racb/ubuntu/+source/memcached:upstream-bump-1.5.6 | ||||||||
Merge into: | ubuntu/+source/memcached:ubuntu/devel | ||||||||
Diff against target: |
1554 lines (+515/-267) 28 files modified
assoc.c (+8/-20) assoc.h (+1/-0) configure (+10/-10) crawler.c (+7/-5) crawler.h (+4/-1) debian/changelog (+10/-0) debian/patches/02_service_wrapper.patch (+6/-8) debian/patches/restore-systemd-sandboxing (+61/-0) debian/patches/series (+1/-1) debian/rules (+2/-0) dev/null (+0/-64) doc/Makefile (+3/-3) items.c (+6/-1) items.h (+1/-0) memcached.c (+28/-9) memcached.spec (+101/-27) memcached_dtrace.d (+2/-6) scripts/memcached-automove-extstore (+25/-30) scripts/memcached.service (+46/-34) scripts/memcached.sysconfig (+10/-0) scripts/memcached@.service (+89/-0) slab_automove_extstore.c (+65/-35) slabs.c (+8/-3) t/issue_67.t (+2/-2) thread.c (+11/-0) timedrun.c (+5/-5) trace.h (+2/-2) version.m4 (+1/-1) |
||||||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Christian Ehrhardt (community) | Approve | ||
Canonical Server | Pending | ||
Review via email: mp+341340@code.launchpad.net |
Commit message
Description of the change
Christian Ehrhardt (paelzer) wrote : | # |
Christian Ehrhardt (paelzer) wrote : | # |
1. verified the upstream delta between version - ok
Christian Ehrhardt (paelzer) wrote : | # |
2. is it really a bugfix release - ok
https:/
https:/
- UDP - we had disabled that already before, so ok
- ##secure## in service file - handled by you (will check in detail) - ok
Also their git log looks sane.
Christian Ehrhardt (paelzer) wrote : | # |
3. changelog - unsure
That could have followed the usual "merge with X, remaining changes .., Dropped changes",
but it is short and complete
For example we still have:
- debian/NEWS: add explanation and document how to re-enable UDP if necessary.
- rules: Disable tests on armhf, they hang the build.
- added patch to show distribution on version
- Force alignment on in configure.ac
- disable unreliable test.
I think it would be good to have the usual "merge" changelog structure in this case as well.
This is not a hard nack, but I want to hear your opinion on it.
Robie Basak (racb) wrote : | # |
It's not a merge. It's a pure upstream release bump. I haven't pulled in any changes from Debian. So the previous changes from 1.5.4-1 in the changelog are current and read correctly as linear history.
Christian Ehrhardt (paelzer) wrote : | # |
4. checked all new commits - ok
Thanks for the pre-discussion on the sandboxing, certainly better to underatand.
The check seems good if they add more.
If they remove some the patch will not apply.
So in both cases a packager will consciously decide then - that is good.
Robie Basak (racb) wrote : | # |
IOW, it's as if I'm uploading a 1.5.4-1ubuntu4 - but since I also update the upstream version, the package version string becomes 1.5.6-0ubuntu1.
Christian Ehrhardt (paelzer) wrote : | # |
5. Commit structure - ok
I'd have preferred to have
1. Latest Debian
2. Upstream diff
3. old Ubuntu changes as-is
4. new Ubuntu changes by you
But that is a matter of personal taste I assume, fine as it is.
Christian Ehrhardt (paelzer) wrote : | # |
6. Did an upgrade and install from the ppa - all ok
Service still seems started the same way (checked as there were the upstream changes)
Christian Ehrhardt (paelzer) wrote : | # |
7. ran the 11 tests of qa-regression-
One fail, but that it had with the former version as well.
Christian Ehrhardt (paelzer) wrote : | # |
Thanks for the explanations on the changelog structure btw.
Overall this LGTM +1.
Preview Diff
1 | diff --git a/assoc.c b/assoc.c |
2 | index 8cdf581..8e7a810 100644 |
3 | --- a/assoc.c |
4 | +++ b/assoc.c |
5 | @@ -27,7 +27,6 @@ |
6 | |
7 | static pthread_cond_t maintenance_cond = PTHREAD_COND_INITIALIZER; |
8 | static pthread_mutex_t maintenance_lock = PTHREAD_MUTEX_INITIALIZER; |
9 | -static pthread_mutex_t hash_items_counter_lock = PTHREAD_MUTEX_INITIALIZER; |
10 | |
11 | typedef unsigned long int ub4; /* unsigned 4-byte quantities */ |
12 | typedef unsigned char ub1; /* unsigned 1-byte quantities */ |
13 | @@ -47,9 +46,6 @@ static item** primary_hashtable = 0; |
14 | */ |
15 | static item** old_hashtable = 0; |
16 | |
17 | -/* Number of items in the hash table. */ |
18 | -static unsigned int hash_items = 0; |
19 | - |
20 | /* Flag: Are we in the middle of expanding now? */ |
21 | static bool expanding = false; |
22 | static bool started_expanding = false; |
23 | @@ -144,12 +140,15 @@ static void assoc_expand(void) { |
24 | } |
25 | } |
26 | |
27 | -static void assoc_start_expand(void) { |
28 | +void assoc_start_expand(uint64_t curr_items) { |
29 | if (started_expanding) |
30 | return; |
31 | |
32 | - started_expanding = true; |
33 | - pthread_cond_signal(&maintenance_cond); |
34 | + if (curr_items > (hashsize(hashpower) * 3) / 2 && |
35 | + hashpower < HASHPOWER_MAX) { |
36 | + started_expanding = true; |
37 | + pthread_cond_signal(&maintenance_cond); |
38 | + } |
39 | } |
40 | |
41 | /* Note: this isn't an assoc_update. The key must not already exist to call this */ |
42 | @@ -168,15 +167,7 @@ int assoc_insert(item *it, const uint32_t hv) { |
43 | primary_hashtable[hv & hashmask(hashpower)] = it; |
44 | } |
45 | |
46 | - pthread_mutex_lock(&hash_items_counter_lock); |
47 | - hash_items++; |
48 | - if (! expanding && hash_items > (hashsize(hashpower) * 3) / 2 && |
49 | - hashpower < HASHPOWER_MAX) { |
50 | - assoc_start_expand(); |
51 | - } |
52 | - pthread_mutex_unlock(&hash_items_counter_lock); |
53 | - |
54 | - MEMCACHED_ASSOC_INSERT(ITEM_key(it), it->nkey, hash_items); |
55 | + MEMCACHED_ASSOC_INSERT(ITEM_key(it), it->nkey); |
56 | return 1; |
57 | } |
58 | |
59 | @@ -185,13 +176,10 @@ void assoc_delete(const char *key, const size_t nkey, const uint32_t hv) { |
60 | |
61 | if (*before) { |
62 | item *nxt; |
63 | - pthread_mutex_lock(&hash_items_counter_lock); |
64 | - hash_items--; |
65 | - pthread_mutex_unlock(&hash_items_counter_lock); |
66 | /* The DTrace probe cannot be triggered as the last instruction |
67 | * due to possible tail-optimization by the compiler |
68 | */ |
69 | - MEMCACHED_ASSOC_DELETE(key, nkey, hash_items); |
70 | + MEMCACHED_ASSOC_DELETE(key, nkey); |
71 | nxt = (*before)->h_next; |
72 | (*before)->h_next = 0; /* probably pointless, but whatever. */ |
73 | *before = nxt; |
74 | diff --git a/assoc.h b/assoc.h |
75 | index dda39a0..fb2c922 100644 |
76 | --- a/assoc.h |
77 | +++ b/assoc.h |
78 | @@ -6,5 +6,6 @@ void assoc_delete(const char *key, const size_t nkey, const uint32_t hv); |
79 | void do_assoc_move_next_bucket(void); |
80 | int start_assoc_maintenance_thread(void); |
81 | void stop_assoc_maintenance_thread(void); |
82 | +void assoc_start_expand(uint64_t curr_items); |
83 | extern unsigned int hashpower; |
84 | extern unsigned int item_lock_hashpower; |
85 | diff --git a/configure b/configure |
86 | index f1e61ff..e479e60 100755 |
87 | --- a/configure |
88 | +++ b/configure |
89 | @@ -1,6 +1,6 @@ |
90 | #! /bin/sh |
91 | # Guess values for system-dependent variables and create Makefiles. |
92 | -# Generated by GNU Autoconf 2.69 for memcached 1.5.4. |
93 | +# Generated by GNU Autoconf 2.69 for memcached 1.5.6. |
94 | # |
95 | # Report bugs to <memcached@googlegroups.com>. |
96 | # |
97 | @@ -580,8 +580,8 @@ MAKEFLAGS= |
98 | # Identity of this package. |
99 | PACKAGE_NAME='memcached' |
100 | PACKAGE_TARNAME='memcached' |
101 | -PACKAGE_VERSION='1.5.4' |
102 | -PACKAGE_STRING='memcached 1.5.4' |
103 | +PACKAGE_VERSION='1.5.6' |
104 | +PACKAGE_STRING='memcached 1.5.6' |
105 | PACKAGE_BUGREPORT='memcached@googlegroups.com' |
106 | PACKAGE_URL='' |
107 | |
108 | @@ -1320,7 +1320,7 @@ if test "$ac_init_help" = "long"; then |
109 | # Omit some internal or obsolete options to make the list less imposing. |
110 | # This message is too long to be a string in the A/UX 3.1 sh. |
111 | cat <<_ACEOF |
112 | -\`configure' configures memcached 1.5.4 to adapt to many kinds of systems. |
113 | +\`configure' configures memcached 1.5.6 to adapt to many kinds of systems. |
114 | |
115 | Usage: $0 [OPTION]... [VAR=VALUE]... |
116 | |
117 | @@ -1391,7 +1391,7 @@ fi |
118 | |
119 | if test -n "$ac_init_help"; then |
120 | case $ac_init_help in |
121 | - short | recursive ) echo "Configuration of memcached 1.5.4:";; |
122 | + short | recursive ) echo "Configuration of memcached 1.5.6:";; |
123 | esac |
124 | cat <<\_ACEOF |
125 | |
126 | @@ -1495,7 +1495,7 @@ fi |
127 | test -n "$ac_init_help" && exit $ac_status |
128 | if $ac_init_version; then |
129 | cat <<\_ACEOF |
130 | -memcached configure 1.5.4 |
131 | +memcached configure 1.5.6 |
132 | generated by GNU Autoconf 2.69 |
133 | |
134 | Copyright (C) 2012 Free Software Foundation, Inc. |
135 | @@ -1964,7 +1964,7 @@ cat >config.log <<_ACEOF |
136 | This file contains any messages produced by compilers while |
137 | running configure, to aid debugging if configure makes a mistake. |
138 | |
139 | -It was created by memcached $as_me 1.5.4, which was |
140 | +It was created by memcached $as_me 1.5.6, which was |
141 | generated by GNU Autoconf 2.69. Invocation command line was |
142 | |
143 | $ $0 $@ |
144 | @@ -2899,7 +2899,7 @@ fi |
145 | |
146 | # Define the identity of the package. |
147 | PACKAGE='memcached' |
148 | - VERSION='1.5.4' |
149 | + VERSION='1.5.6' |
150 | |
151 | |
152 | cat >>confdefs.h <<_ACEOF |
153 | @@ -7156,7 +7156,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 |
154 | # report actual input values of CONFIG_FILES etc. instead of their |
155 | # values after options handling. |
156 | ac_log=" |
157 | -This file was extended by memcached $as_me 1.5.4, which was |
158 | +This file was extended by memcached $as_me 1.5.6, which was |
159 | generated by GNU Autoconf 2.69. Invocation command line was |
160 | |
161 | CONFIG_FILES = $CONFIG_FILES |
162 | @@ -7222,7 +7222,7 @@ _ACEOF |
163 | cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 |
164 | ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" |
165 | ac_cs_version="\\ |
166 | -memcached config.status 1.5.4 |
167 | +memcached config.status 1.5.6 |
168 | configured by $0, generated by GNU Autoconf 2.69, |
169 | with options \\"\$ac_cs_config\\" |
170 | |
171 | diff --git a/crawler.c b/crawler.c |
172 | index 56e967a..9c81bba 100644 |
173 | --- a/crawler.c |
174 | +++ b/crawler.c |
175 | @@ -252,8 +252,8 @@ static void crawler_metadump_eval(crawler_module_t *cm, item *it, uint32_t hv, i |
176 | int total = snprintf(cm->c.cbuf, 4096, |
177 | "key=%s exp=%ld la=%llu cas=%llu fetch=%s cls=%u size=%lu\n", |
178 | keybuf, |
179 | - (it->exptime == 0) ? -1 : (long)it->exptime + process_started, |
180 | - (unsigned long long)it->time + process_started, |
181 | + (it->exptime == 0) ? -1 : (long)(it->exptime + process_started), |
182 | + (unsigned long long)(it->time + process_started), |
183 | (unsigned long long)ITEM_get_cas(it), |
184 | (it->it_flags & ITEM_FETCHED) ? "yes" : "no", |
185 | ITEM_clsid(it), |
186 | @@ -530,6 +530,9 @@ static int do_lru_crawler_start(uint32_t id, uint32_t remaining) { |
187 | crawlers[sid].next = 0; |
188 | crawlers[sid].prev = 0; |
189 | crawlers[sid].time = 0; |
190 | + if (remaining == LRU_CRAWLER_CAP_REMAINING) { |
191 | + remaining = do_get_lru_size(sid); |
192 | + } |
193 | crawlers[sid].remaining = remaining; |
194 | crawlers[sid].slabs_clsid = sid; |
195 | crawlers[sid].reclaimed = 0; |
196 | @@ -622,7 +625,7 @@ int lru_crawler_start(uint8_t *ids, uint32_t remaining, |
197 | * Also only clear the crawlerstats once per sid. |
198 | */ |
199 | enum crawler_result_type lru_crawler_crawl(char *slabs, const enum crawler_run_type type, |
200 | - void *c, const int sfd) { |
201 | + void *c, const int sfd, unsigned int remaining) { |
202 | char *b = NULL; |
203 | uint32_t sid = 0; |
204 | int starts = 0; |
205 | @@ -651,8 +654,7 @@ enum crawler_result_type lru_crawler_crawl(char *slabs, const enum crawler_run_t |
206 | } |
207 | } |
208 | |
209 | - starts = lru_crawler_start(tocrawl, settings.lru_crawler_tocrawl, |
210 | - type, NULL, c, sfd); |
211 | + starts = lru_crawler_start(tocrawl, remaining, type, NULL, c, sfd); |
212 | if (starts == -1) { |
213 | return CRAWLER_RUNNING; |
214 | } else if (starts == -2) { |
215 | diff --git a/crawler.h b/crawler.h |
216 | index e009548..f7ebba2 100644 |
217 | --- a/crawler.h |
218 | +++ b/crawler.h |
219 | @@ -1,6 +1,8 @@ |
220 | #ifndef CRAWLER_H |
221 | #define CRAWLER_H |
222 | |
223 | +#define LRU_CRAWLER_CAP_REMAINING -1 |
224 | + |
225 | typedef struct { |
226 | uint64_t histo[61]; |
227 | uint64_t ttl_hourplus; |
228 | @@ -28,7 +30,8 @@ enum crawler_result_type { |
229 | int start_item_crawler_thread(void); |
230 | int stop_item_crawler_thread(void); |
231 | int init_lru_crawler(void *arg); |
232 | -enum crawler_result_type lru_crawler_crawl(char *slabs, enum crawler_run_type, void *c, const int sfd); |
233 | +enum crawler_result_type lru_crawler_crawl(char *slabs, enum crawler_run_type, |
234 | + void *c, const int sfd, unsigned int remaining); |
235 | int lru_crawler_start(uint8_t *ids, uint32_t remaining, |
236 | const enum crawler_run_type type, void *data, |
237 | void *c, const int sfd); |
238 | diff --git a/debian/changelog b/debian/changelog |
239 | index 9914ad2..f1ae7d9 100644 |
240 | --- a/debian/changelog |
241 | +++ b/debian/changelog |
242 | @@ -1,3 +1,13 @@ |
243 | +memcached (1.5.6-0ubuntu1) bionic; urgency=medium |
244 | + |
245 | + * New upstream bugfix release (LP: #1753839). |
246 | + * d/p/disable-udp-by-default.patch: drop (now upstream). |
247 | + * d/p/02_service_wrapper.patch: refresh to remove fuzz. |
248 | + * d/p/restore-systemd-sandboxing: restore sandboxing in memcached.service |
249 | + removed by upstream in 1.5.6 to avoid feature regression (LP: #1755460). |
250 | + |
251 | + -- Robie Basak <robie.basak@ubuntu.com> Tue, 13 Mar 2018 09:59:06 +0000 |
252 | + |
253 | memcached (1.5.4-1ubuntu3) bionic; urgency=medium |
254 | |
255 | * SECURITY UPDATE: disable listening on UDP port by default due to |
256 | diff --git a/debian/patches/02_service_wrapper.patch b/debian/patches/02_service_wrapper.patch |
257 | index a81bd06..355e194 100644 |
258 | --- a/debian/patches/02_service_wrapper.patch |
259 | +++ b/debian/patches/02_service_wrapper.patch |
260 | @@ -7,10 +7,8 @@ Description: Additions for service unit file |
261 | Last-Update: 2017-09-13 |
262 | Forwarded: not-needed |
263 | |
264 | -Index: memcached/scripts/memcached.service |
265 | -=================================================================== |
266 | ---- memcached.orig/scripts/memcached.service |
267 | -+++ memcached/scripts/memcached.service |
268 | +--- a/scripts/memcached.service |
269 | ++++ b/scripts/memcached.service |
270 | @@ -12,10 +12,10 @@ |
271 | [Unit] |
272 | Description=memcached daemon |
273 | @@ -22,11 +20,11 @@ Index: memcached/scripts/memcached.service |
274 | -ExecStart=/usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS |
275 | +ExecStart=/usr/share/memcached/scripts/systemd-memcached-wrapper /etc/memcached.conf |
276 | |
277 | - # Set up a new file system namespace and mounts private /tmp and /var/tmp directories |
278 | - # so this service cannot access the global directories and other processes cannot |
279 | -@@ -68,5 +68,10 @@ RestrictAddressFamilies=AF_INET AF_INET6 |
280 | + # Set up a new file system namespace and mounts private /tmp and /var/tmp |
281 | + # directories so this service cannot access the global directories and |
282 | +@@ -80,5 +80,10 @@ |
283 | # Takes away the ability to create or manage any kind of namespace |
284 | - RestrictNamespaces=true |
285 | + ##safer##RestrictNamespaces=true |
286 | |
287 | +PIDFile=/var/run/memcached/memcached.pid |
288 | + |
289 | diff --git a/debian/patches/disable-udp-by-default.patch b/debian/patches/disable-udp-by-default.patch |
290 | deleted file mode 100644 |
291 | index 7ca2d13..0000000 |
292 | --- a/debian/patches/disable-udp-by-default.patch |
293 | +++ /dev/null |
294 | @@ -1,64 +0,0 @@ |
295 | -From dbb7a8af90054bf4ef51f5814ef7ceb17d83d974 Mon Sep 17 00:00:00 2001 |
296 | -From: dormando <dormando@rydia.net> |
297 | -Date: Tue, 27 Feb 2018 10:50:45 -0800 |
298 | -Subject: [PATCH] disable UDP port by default |
299 | - |
300 | -As reported, UDP amplification attacks have started to use insecure |
301 | -internet-exposed memcached instances. UDP used to be a lot more popular as a |
302 | -transport for memcached many years ago, but I'm not aware of many recent |
303 | -users. |
304 | - |
305 | -Ten years ago, the TCP connection overhead from many clients was relatively |
306 | -high (dozens or hundreds per client server), but these days many clients are |
307 | -batched, or user fewer processes, or simply anre't worried about it. |
308 | - |
309 | -While changing the default to listen on localhost only would also help, the |
310 | -true culprit is UDP. There are many more use cases for using memcached over |
311 | -the network than there are for using the UDP protocol. |
312 | ---- |
313 | - memcached.c | 6 ++---- |
314 | - t/issue_67.t | 4 ++-- |
315 | - 2 files changed, 4 insertions(+), 6 deletions(-) |
316 | - |
317 | -diff --git a/memcached.c b/memcached.c |
318 | -index 88a5f2ebe..7178666e2 100644 |
319 | ---- a/memcached.c |
320 | -+++ b/memcached.c |
321 | -@@ -222,7 +222,7 @@ static void settings_init(void) { |
322 | - settings.use_cas = true; |
323 | - settings.access = 0700; |
324 | - settings.port = 11211; |
325 | -- settings.udpport = 11211; |
326 | -+ settings.udpport = 0; |
327 | - /* By default this string should be NULL for getaddrinfo() */ |
328 | - settings.inter = NULL; |
329 | - settings.maxbytes = 64 * 1024 * 1024; /* default is 64MB */ |
330 | -@@ -7479,9 +7479,7 @@ int main (int argc, char **argv) { |
331 | - } |
332 | - } |
333 | - |
334 | -- if (tcp_specified && settings.port != 0 && !udp_specified) { |
335 | -- settings.udpport = settings.port; |
336 | -- } else if (udp_specified && settings.udpport != 0 && !tcp_specified) { |
337 | -+ if (udp_specified && settings.udpport != 0 && !tcp_specified) { |
338 | - settings.port = settings.udpport; |
339 | - } |
340 | - |
341 | -diff --git a/t/issue_67.t b/t/issue_67.t |
342 | -index 69c869f4c..b2d374fa5 100644 |
343 | ---- a/t/issue_67.t |
344 | -+++ b/t/issue_67.t |
345 | -@@ -77,11 +77,11 @@ sub when { |
346 | - |
347 | - # Disabling the defaults since it conflicts with a running instance. |
348 | - # when('no arguments', '', 11211, 11211); |
349 | --when('specifying tcp port', '-p 11212', 11212, 11212); |
350 | -+when('specifying tcp port', '-p 11212', 11212, -1); |
351 | - when('specifying udp port', '-U 11222', 11222, 11222); |
352 | - when('specifying tcp ephemeral port', '-p -1', 0, 0); |
353 | - when('specifying udp ephemeral port', '-U -1', 0, 0); |
354 | --when('tcp port disabled', '-p 0', -1, 11211); |
355 | -+when('tcp port disabled', '-p 0', -1, -1); |
356 | - when('udp port disabled', '-U 0', 11211, -1); |
357 | - when('specifying tcp and udp ports', '-p 11232 -U 11233', 11232, 11233); |
358 | - when('specifying tcp and disabling udp', '-p 11242 -U 0', 11242, -1); |
359 | diff --git a/debian/patches/restore-systemd-sandboxing b/debian/patches/restore-systemd-sandboxing |
360 | new file mode 100644 |
361 | index 0000000..584e774 |
362 | --- /dev/null |
363 | +++ b/debian/patches/restore-systemd-sandboxing |
364 | @@ -0,0 +1,61 @@ |
365 | +Author: Robie Basak <robie.basak@canonical.com> |
366 | +Description: Restore systemd sandboxing |
367 | + Upstream regressed systemd sandboxing for everyone by default because RHEL |
368 | + cannot support it. Put it back again to avoid this functional regression. |
369 | +Bug: https://github.com/memcached/memcached/issues/359 |
370 | +Bug-Ubuntu: https://bugs.launchpad.net/memcached/+bug/1755460 |
371 | +Forwarded: not-needed |
372 | +Last-Update: 2018-03-13 |
373 | + |
374 | +--- a/scripts/memcached.service |
375 | ++++ b/scripts/memcached.service |
376 | +@@ -42,21 +42,16 @@ |
377 | + # of this unit. Protects against vulnerabilities such as CVE-2016-8655 |
378 | + RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
379 | + |
380 | +- |
381 | +-# Some security features are not in the older versions of systemd used by |
382 | +-# e.g. RHEL7/CentOS 7. The below settings are automatically edited at package |
383 | +-# build time to uncomment them if the target platform supports them. |
384 | +- |
385 | + # Attempts to create memory mappings that are writable and executable at |
386 | + # the same time, or to change existing memory mappings to become executable |
387 | + # are prohibited. |
388 | +-##safer##MemoryDenyWriteExecute=true |
389 | ++MemoryDenyWriteExecute=true |
390 | + |
391 | + # Explicit module loading will be denied. This allows to turn off module |
392 | + # load and unload operations on modular kernels. It is recommended to turn |
393 | + # this on for most services that do not need special file systems or extra |
394 | + # kernel modules to work. |
395 | +-##safer##ProtectKernelModules=true |
396 | ++ProtectKernelModules=true |
397 | + |
398 | + # Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, |
399 | + # /proc/latency_stats, /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq |
400 | +@@ -64,21 +59,21 @@ |
401 | + # kernel variables should only be written at boot-time, with the sysctl.d(5) |
402 | + # mechanism. Almost no services need to write to these at runtime; it is hence |
403 | + # recommended to turn this on for most services. |
404 | +-##safer##ProtectKernelTunables=true |
405 | ++ProtectKernelTunables=true |
406 | + |
407 | + # The Linux Control Groups (cgroups(7)) hierarchies accessible through |
408 | + # /sys/fs/cgroup will be made read-only to all processes of the unit. |
409 | + # Except for container managers no services should require write access |
410 | + # to the control groups hierarchies; it is hence recommended to turn this |
411 | + # on for most services |
412 | +-##safer##ProtectControlGroups=true |
413 | ++ProtectControlGroups=true |
414 | + |
415 | + # Any attempts to enable realtime scheduling in a process of the unit are |
416 | + # refused. |
417 | +-##safer##RestrictRealtime=true |
418 | ++RestrictRealtime=true |
419 | + |
420 | + # Takes away the ability to create or manage any kind of namespace |
421 | +-##safer##RestrictNamespaces=true |
422 | ++RestrictNamespaces=true |
423 | + |
424 | + PIDFile=/var/run/memcached/memcached.pid |
425 | + |
426 | diff --git a/debian/patches/series b/debian/patches/series |
427 | index 4e08bec..bb9c45b 100644 |
428 | --- a/debian/patches/series |
429 | +++ b/debian/patches/series |
430 | @@ -5,4 +5,4 @@ |
431 | fix-distribution.patch |
432 | always_enable_alignment.patch |
433 | disable_watcher_test.patch |
434 | -disable-udp-by-default.patch |
435 | +restore-systemd-sandboxing |
436 | diff --git a/debian/rules b/debian/rules |
437 | index ccd01ec..9fc8d04 100755 |
438 | --- a/debian/rules |
439 | +++ b/debian/rules |
440 | @@ -26,6 +26,8 @@ override_dh_auto_install: |
441 | $(CURDIR)/debian/memcached.init |
442 | install -m 755 $(CURDIR)/scripts/memcached.service \ |
443 | $(CURDIR)/debian/memcached.service |
444 | + # Check for LP: #1755460 |
445 | + if grep -i '##safer##' $(CURDIR)/debian/memcached.service >/dev/null 2>&1; then echo "ERROR: debian/patches/restore-systemd-sandboxing is incomplete; please see LP: #1755460" >&2; exit 1; fi |
446 | install -m 755 $(CURDIR)/scripts/damemtop $(SCRIPTS) |
447 | install -m 644 $(CURDIR)/scripts/damemtop.yaml $(SCRIPTS) |
448 | install -m 644 $(CURDIR)/scripts/README.damemtop $(SCRIPTS) |
449 | diff --git a/doc/Makefile b/doc/Makefile |
450 | index 881c525..0337680 100644 |
451 | --- a/doc/Makefile |
452 | +++ b/doc/Makefile |
453 | @@ -191,10 +191,10 @@ OBJEXT = o |
454 | PACKAGE = memcached |
455 | PACKAGE_BUGREPORT = memcached@googlegroups.com |
456 | PACKAGE_NAME = memcached |
457 | -PACKAGE_STRING = memcached 1.5.4 |
458 | +PACKAGE_STRING = memcached 1.5.6 |
459 | PACKAGE_TARNAME = memcached |
460 | PACKAGE_URL = |
461 | -PACKAGE_VERSION = 1.5.4 |
462 | +PACKAGE_VERSION = 1.5.6 |
463 | PATH_SEPARATOR = : |
464 | PROFILER = /usr/bin/gcov |
465 | PROFILER_FLAGS = -fprofile-arcs -ftest-coverage |
466 | @@ -202,7 +202,7 @@ PROFILER_LDFLAGS = -lgcov |
467 | SET_MAKE = |
468 | SHELL = /bin/bash |
469 | STRIP = |
470 | -VERSION = 1.5.4 |
471 | +VERSION = 1.5.6 |
472 | XML2RFC = /usr/bin/xml2rfc |
473 | XSLTPROC = no |
474 | abs_builddir = /home/dormando/d/p/danga/git/memcached/doc |
475 | diff --git a/items.c b/items.c |
476 | index 400048b..0aefaf0 100644 |
477 | --- a/items.c |
478 | +++ b/items.c |
479 | @@ -138,6 +138,11 @@ static unsigned int temp_lru_size(int slabs_clsid) { |
480 | return ret; |
481 | } |
482 | |
483 | +/* must be locked before call */ |
484 | +unsigned int do_get_lru_size(uint32_t id) { |
485 | + return sizes[id]; |
486 | +} |
487 | + |
488 | /* Enable this for reference-count debugging. */ |
489 | #if 0 |
490 | # define DEBUG_REFCNT(it,op) \ |
491 | @@ -925,7 +930,7 @@ void item_stats_sizes(ADD_STAT add_stats, void *c) { |
492 | int i; |
493 | for (i = 0; i < stats_sizes_buckets; i++) { |
494 | if (stats_sizes_hist[i] != 0) { |
495 | - char key[8]; |
496 | + char key[12]; |
497 | snprintf(key, sizeof(key), "%d", i * 32); |
498 | APPEND_STAT(key, "%u", stats_sizes_hist[i]); |
499 | } |
500 | diff --git a/items.h b/items.h |
501 | index bf126d7..3b99c66 100644 |
502 | --- a/items.h |
503 | +++ b/items.h |
504 | @@ -25,6 +25,7 @@ void do_item_update_nolock(item *it); |
505 | int do_item_replace(item *it, item *new_it, const uint32_t hv); |
506 | |
507 | int item_is_flushed(item *it); |
508 | +unsigned int do_get_lru_size(uint32_t id); |
509 | |
510 | void do_item_linktail_q(item *it); |
511 | void do_item_unlinktail_q(item *it); |
512 | diff --git a/memcached.c b/memcached.c |
513 | index a8a724c..7178666 100644 |
514 | --- a/memcached.c |
515 | +++ b/memcached.c |
516 | @@ -222,7 +222,7 @@ static void settings_init(void) { |
517 | settings.use_cas = true; |
518 | settings.access = 0700; |
519 | settings.port = 11211; |
520 | - settings.udpport = 11211; |
521 | + settings.udpport = 0; |
522 | /* By default this string should be NULL for getaddrinfo() */ |
523 | settings.inter = NULL; |
524 | settings.maxbytes = 64 * 1024 * 1024; /* default is 64MB */ |
525 | @@ -3338,7 +3338,7 @@ static void process_stats_conns(ADD_STAT add_stats, void *c) { |
526 | int i; |
527 | char key_str[STAT_KEY_LEN]; |
528 | char val_str[STAT_VAL_LEN]; |
529 | - char conn_name[MAXPATHLEN + sizeof("unix:")]; |
530 | + char conn_name[MAXPATHLEN + sizeof("unix:") + sizeof("65535")]; |
531 | int klen = 0, vlen = 0; |
532 | |
533 | assert(add_stats); |
534 | @@ -4525,7 +4525,7 @@ static void process_lru_command(conn *c, token_t *tokens, const size_t ntokens) |
535 | static void process_extstore_command(conn *c, token_t *tokens, const size_t ntokens) { |
536 | set_noreply_maybe(c, tokens, ntokens); |
537 | bool ok = true; |
538 | - if (ntokens < 3) { |
539 | + if (ntokens < 4) { |
540 | ok = false; |
541 | } else if (strcmp(tokens[1].value, "free_memchunks") == 0 && ntokens > 4) { |
542 | /* per-slab-class free chunk setting. */ |
543 | @@ -4770,7 +4770,8 @@ static void process_command(conn *c, char *command) { |
544 | return; |
545 | } |
546 | |
547 | - rv = lru_crawler_crawl(tokens[2].value, CRAWLER_EXPIRED, NULL, 0); |
548 | + rv = lru_crawler_crawl(tokens[2].value, CRAWLER_EXPIRED, NULL, 0, |
549 | + settings.lru_crawler_tocrawl); |
550 | switch(rv) { |
551 | case CRAWLER_OK: |
552 | out_string(c, "OK"); |
553 | @@ -4800,7 +4801,7 @@ static void process_command(conn *c, char *command) { |
554 | } |
555 | |
556 | int rv = lru_crawler_crawl(tokens[2].value, CRAWLER_METADUMP, |
557 | - c, c->sfd); |
558 | + c, c->sfd, LRU_CRAWLER_CAP_REMAINING); |
559 | switch(rv) { |
560 | case CRAWLER_OK: |
561 | out_string(c, "OK"); |
562 | @@ -5973,6 +5974,7 @@ static int server_sockets(int port, enum network_transport transport, |
563 | char *e = strchr(p, ']'); |
564 | if (e == NULL) { |
565 | fprintf(stderr, "Invalid IPV6 address: \"%s\"", p); |
566 | + free(list); |
567 | return 1; |
568 | } |
569 | h = ++p; // skip the opening '[' |
570 | @@ -5991,6 +5993,7 @@ static int server_sockets(int port, enum network_transport transport, |
571 | ++s; |
572 | if (!safe_strtol(s, &the_port)) { |
573 | fprintf(stderr, "Invalid port number: \"%s\"", s); |
574 | + free(list); |
575 | return 1; |
576 | } |
577 | } |
578 | @@ -6126,6 +6129,10 @@ static void clock_handler(const int fd, const short which, void *arg) { |
579 | #endif |
580 | } |
581 | |
582 | + // While we're here, check for hash table expansion. |
583 | + // This function should be quick to avoid delaying the timer. |
584 | + assoc_start_expand(stats_state.curr_items); |
585 | + |
586 | evtimer_set(&clockevent, clock_handler, 0); |
587 | event_base_set(main_base, &clockevent); |
588 | evtimer_add(&clockevent, &t); |
589 | @@ -7472,9 +7479,7 @@ int main (int argc, char **argv) { |
590 | } |
591 | } |
592 | |
593 | - if (tcp_specified && settings.port != 0 && !udp_specified) { |
594 | - settings.udpport = settings.port; |
595 | - } else if (udp_specified && settings.udpport != 0 && !tcp_specified) { |
596 | + if (udp_specified && settings.udpport != 0 && !tcp_specified) { |
597 | settings.port = settings.udpport; |
598 | } |
599 | |
600 | @@ -7568,7 +7573,17 @@ int main (int argc, char **argv) { |
601 | } |
602 | |
603 | /* initialize main thread libevent instance */ |
604 | +#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= 0x02000101 |
605 | + /* If libevent version is larger/equal to 2.0.2-alpha, use newer version */ |
606 | + struct event_config *ev_config; |
607 | + ev_config = event_config_new(); |
608 | + event_config_set_flag(ev_config, EVENT_BASE_FLAG_NOLOCK); |
609 | + main_base = event_base_new_with_config(ev_config); |
610 | + event_config_free(ev_config); |
611 | +#else |
612 | + /* Otherwise, use older API */ |
613 | main_base = event_init(); |
614 | +#endif |
615 | |
616 | /* initialize other stuff */ |
617 | logger_init(); |
618 | @@ -7710,8 +7725,9 @@ int main (int argc, char **argv) { |
619 | if (portnumber_file) { |
620 | fclose(portnumber_file); |
621 | rename(temp_portnumber_filename, portnumber_filename); |
622 | - free(temp_portnumber_filename); |
623 | } |
624 | + if (temp_portnumber_filename) |
625 | + free(temp_portnumber_filename); |
626 | } |
627 | |
628 | /* Give the sockets a moment to open. I know this is dumb, but the error |
629 | @@ -7753,5 +7769,8 @@ int main (int argc, char **argv) { |
630 | if (u_socket) |
631 | free(u_socket); |
632 | |
633 | + /* cleanup base */ |
634 | + event_base_free(main_base); |
635 | + |
636 | return retval; |
637 | } |
638 | diff --git a/memcached.spec b/memcached.spec |
639 | index a149db3..697c38f 100644 |
640 | --- a/memcached.spec |
641 | +++ b/memcached.spec |
642 | @@ -1,21 +1,48 @@ |
643 | +# Set with_systemd on distros that use it, so we can install the service |
644 | +# file, otherwise the sysvinit script will be installed |
645 | +%if 0%{?fedora} >= 14 || 0%{?rhel} >= 7 || 0%{?suse_version} >= 1210 |
646 | +%global with_systemd 1 |
647 | +BuildRequires: systemd-units |
648 | + |
649 | +# Disable some systemd safety features on OSes without a new enough systemd |
650 | +# (new enough is systemd >= 233) |
651 | +%if 0%{?fedora} < 26 || 0%{?rhel} > 0 |
652 | +%global safer_systemd 0 |
653 | +%else |
654 | +%global safer_systemd 1 |
655 | +%endif |
656 | + |
657 | +%else |
658 | +%global with_systemd 0 |
659 | +%endif |
660 | + |
661 | Name: memcached |
662 | -Version: 1.5.4 |
663 | +Version: 1.5.6 |
664 | Release: 1%{?dist} |
665 | Summary: High Performance, Distributed Memory Object Cache |
666 | |
667 | Group: System Environment/Daemons |
668 | License: BSD |
669 | URL: http://memcached.org |
670 | -Source0: http://memcached.org/files/%{name}-1.5.4.tar.gz |
671 | +Source0: http://memcached.org/files/%{name}-%{version}.tar.gz |
672 | +Source1: memcached.sysconfig |
673 | +Source2: memcached.service |
674 | +Source3: memcached@.service |
675 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) |
676 | |
677 | BuildRequires: libevent-devel |
678 | BuildRequires: perl(Test::More) |
679 | BuildRequires: /usr/bin/prove |
680 | Requires: initscripts |
681 | +%if %{with_systemd} |
682 | +Requires(post): systemd-units |
683 | +Requires(preun): systemd-units |
684 | +Requires(postun): systemd-units |
685 | +%else |
686 | Requires(post): /sbin/chkconfig |
687 | Requires(preun): /sbin/chkconfig, /sbin/service |
688 | Requires(postun): /sbin/service |
689 | +%endif |
690 | |
691 | %description |
692 | memcached is a high-performance, distributed memory object caching |
693 | @@ -23,7 +50,7 @@ system, generic in nature, but intended for use in speeding up dynamic |
694 | web applications by alleviating database load. |
695 | |
696 | %prep |
697 | -%setup -q -n %{name}-1.5.4 |
698 | +%setup -q -n %{name}-%{version} |
699 | |
700 | |
701 | %build |
702 | @@ -31,53 +58,90 @@ web applications by alleviating database load. |
703 | |
704 | make %{?_smp_mflags} |
705 | |
706 | + |
707 | %check |
708 | make test |
709 | |
710 | + |
711 | %install |
712 | rm -rf %{buildroot} |
713 | make install DESTDIR=%{buildroot} |
714 | |
715 | # remove memcached-debug |
716 | -rm -f %{buildroot}/%{_bindir}/memcached-debug |
717 | +rm -f %{buildroot}/%{_bindir}/%{name}-debug |
718 | |
719 | # Perl script for monitoring memcached |
720 | -install -Dp -m0755 scripts/memcached-tool %{buildroot}%{_bindir}/memcached-tool |
721 | +install -Dp -m0755 scripts/memcached-tool %{buildroot}%{_bindir}/%{name}-tool |
722 | |
723 | # Init script |
724 | -install -Dp -m0755 scripts/memcached.sysv %{buildroot}%{_initrddir}/memcached |
725 | +%if %{with_systemd} |
726 | +install -Dp -m0755 scripts/memcached.service %{buildroot}%{_unitdir}/%{name}.service |
727 | +install -Dp -m0755 scripts/memcached@.service %{buildroot}%{_unitdir}/%{name}@.service |
728 | + |
729 | +if [ %{safer_systemd} -gt 0 ]; then |
730 | + sed -e -i 's/^##safer##//g' %{buildroot}%{_unitdir}/%{name}.service %{buildroot}%{_unitdir}/%{name}@.service |
731 | +else |
732 | + sed -e -i 's/^##safer##/#/g' %{buildroot}%{_unitdir}/%{name}.service %{buildroot}%{_unitdir}/%{name}@.service |
733 | +fi |
734 | +%else |
735 | +install -Dp -m0755 scripts/memcached.sysv %{buildroot}%{_initrddir}/%{name} |
736 | +%endif |
737 | |
738 | # Default configs |
739 | -mkdir -p %{buildroot}/%{_sysconfdir}/sysconfig |
740 | -cat <<EOF >%{buildroot}/%{_sysconfdir}/sysconfig/%{name} |
741 | -PORT="11211" |
742 | -USER="nobody" |
743 | -MAXCONN="1024" |
744 | -CACHESIZE="64" |
745 | -OPTIONS="" |
746 | -EOF |
747 | +install -Dp -m0644 scripts/memcached.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/%{name} |
748 | |
749 | # pid directory |
750 | -mkdir -p %{buildroot}/%{_localstatedir}/run/memcached |
751 | +mkdir -p %{buildroot}/%{_localstatedir}/run/%{name} |
752 | + |
753 | |
754 | %clean |
755 | rm -rf %{buildroot} |
756 | |
757 | |
758 | %post |
759 | -/sbin/chkconfig --add %{name} |
760 | +if [ $1 -eq 1 ]; then |
761 | + # Initial install |
762 | +%if %{with_systemd} |
763 | + /bin/systemctl daemon-reload >/dev/null 2>&1 || : |
764 | +%else |
765 | + /sbin/chkconfig --add %{name} |
766 | +%endif |
767 | +fi |
768 | + |
769 | |
770 | %preun |
771 | if [ "$1" = 0 ] ; then |
772 | - /sbin/service %{name} stop > /dev/null 2>&1 |
773 | + # Removal, not upgrade |
774 | +%if %{with_systemd} |
775 | + /bin/systemctl --no-reload disable %{name}.service > /dev/null 2>&1 || : |
776 | + /bin/systemctl --no-reload disable %{name}@\*.service > /dev/null 2>&1 || : |
777 | + /bin/systemctl stop %{name}.service > /dev/null 2>&1 || : |
778 | + /bin/systemctl stop %{name}@\*.service > /dev/null 2>&1 || : |
779 | +%else |
780 | + /sbin/service %{name} stop > /dev/null 2&>1 || : |
781 | /sbin/chkconfig --del %{name} |
782 | +%endif |
783 | fi |
784 | + |
785 | exit 0 |
786 | |
787 | + |
788 | %postun |
789 | -if [ "$1" -ge 1 ]; then |
790 | - /sbin/service %{name} condrestart > /dev/null 2>&1 |
791 | -fi |
792 | +%if %{with_systemd} |
793 | + /bin/systemctl daemon-reload >/dev/null 2>&1 || : |
794 | +%endif |
795 | + |
796 | +# Don't auto-restart memcached on upgrade -- let user control when cache flushes |
797 | +# if [ "$1" -ge 1 ]; then |
798 | +# # upgrade, not install |
799 | +# %if %{with_systemd} |
800 | +# /bin/systemctl try-restart %{name}.service |
801 | +# /bin/systemctl try-restart %{name}@\*.service |
802 | +# %else |
803 | +# /sbin/service %named condrestart 2>/dev/null || : |
804 | +# %endif |
805 | +#fi |
806 | + |
807 | exit 0 |
808 | |
809 | |
810 | @@ -86,14 +150,24 @@ exit 0 |
811 | %doc AUTHORS ChangeLog COPYING NEWS README.md doc/CONTRIBUTORS doc/*.txt |
812 | %config(noreplace) %{_sysconfdir}/sysconfig/%{name} |
813 | |
814 | -%dir %attr(750,nobody,nobody) %{_localstatedir}/run/memcached |
815 | -%{_bindir}/memcached-tool |
816 | -%{_bindir}/memcached |
817 | -%{_mandir}/man1/memcached.1* |
818 | -%{_initrddir}/memcached |
819 | -%{_includedir}/memcached |
820 | +%dir %attr(750,nobody,nobody) %{_localstatedir}/run/%{name} |
821 | +%{_bindir}/%{name}-tool |
822 | +%{_bindir}/%{name} |
823 | +%{_mandir}/man1/%{name}.1* |
824 | +%{_includedir}/%{name} |
825 | + |
826 | +%if %{with_systemd} |
827 | +%{_unitdir}/%{name}.service |
828 | +%{_unitdir}/%{name}@.service |
829 | +%else |
830 | +%{_initrddir}/%{name} |
831 | +%endif |
832 | |
833 | %changelog |
834 | +* Wed Jul 5 2017 J. Grizzard <jg-github@lupine.org> - 1.4.39 |
835 | +- Add systemd-aware build |
836 | +- Add both static and instanced versions of memcached unit files |
837 | + |
838 | * Mon Nov 2 2009 Dormando <dormando@rydia.net> - 1.4.3-1 |
839 | - Fix autogen more. |
840 | |
841 | @@ -113,7 +187,7 @@ exit 0 |
842 | - above suggestions from Bernard Johnson |
843 | |
844 | * Mon May 7 2007 Paul Lindner <lindner@inuus.com> - 1.2.2-2 |
845 | -- Tidiness improvements suggested by Ruben Kerkhof in bugzilla #238994 |
846 | +- Tidyness improvements suggested by Ruben Kerkhof in bugzilla #238994 |
847 | |
848 | * Fri May 4 2007 Paul Lindner <lindner@inuus.com> - 1.2.2-1 |
849 | - Initial spec file created via rpmdev-newspec |
850 | diff --git a/memcached_dtrace.d b/memcached_dtrace.d |
851 | index 41617ff..3443795 100644 |
852 | --- a/memcached_dtrace.d |
853 | +++ b/memcached_dtrace.d |
854 | @@ -111,19 +111,15 @@ provider memcached { |
855 | * Fired when a new item has been inserted. |
856 | * @param key the key just inserted |
857 | * @param keylen length of the key |
858 | - * @param nokeys the total number of keys currently stored, |
859 | - * including the key for which insert was called. |
860 | */ |
861 | - probe assoc__insert(const char *key, int keylen, int nokeys); |
862 | + probe assoc__insert(const char *key, int keylen); |
863 | |
864 | /** |
865 | * Fired when a new item has been removed. |
866 | * @param key the key just deleted |
867 | * @param keylen length of the key |
868 | - * @param nokeys the total number of keys currently stored, |
869 | - * excluding the key for which delete was called. |
870 | */ |
871 | - probe assoc__delete(const char *key, int keylen, int nokeys); |
872 | + probe assoc__delete(const char *key, int keylen); |
873 | |
874 | /** |
875 | * Fired when an item is linked into the cache. |
876 | diff --git a/scripts/memcached-automove-extstore b/scripts/memcached-automove-extstore |
877 | index fa7a11c..db66d8c 100755 |
878 | --- a/scripts/memcached-automove-extstore |
879 | +++ b/scripts/memcached-automove-extstore |
880 | @@ -55,17 +55,17 @@ def window_key_check(history, key): |
881 | def determine_move(history, stats, diffs, memfree): |
882 | """ Figure out of a page move is in order. |
883 | |
884 | - - we should use as much memory as possible to buffer items, reducing |
885 | - the load on flash. |
886 | + - Use as much memory as possible to hold items, reducing the load on |
887 | + flash. |
888 | - tries to keep the global free page pool inbetween poolmin/poolmax. |
889 | - avoids flapping as much as possible: |
890 | - only pull pages off of a class if it hasn't recently evicted or allocated pages. |
891 | - - only pull pages off if a sufficient number of chunks are available. |
892 | + - only pull pages off if a sufficient number of free chunks are available. |
893 | - if global pool is below minimum remove pages from oldest large class. |
894 | - if global pool is above maximum, move pages to youngest large class. |
895 | - - re-adjusts number of free chunks once per minute, based on usage. |
896 | - - age balancing should move memory around large classes, which then |
897 | - adjusts the per-class free chunk minimums. |
898 | + - extstore manages a desired number of free chunks in each slab class. |
899 | + - autmover adjusts above limits once per minute based on current sizes. |
900 | + - if youngest is below the age ratio limit of oldest, move a page to it. |
901 | """ |
902 | # rotate windows |
903 | history['w'].append({}) |
904 | @@ -88,6 +88,7 @@ def determine_move(history, stats, diffs, memfree): |
905 | pool_high = window_key_check(history, 'slab_pool_high') |
906 | for sid, slab in diffs.items(): |
907 | small_slab = False |
908 | + free_enough = False |
909 | # Only balance larger slab classes |
910 | if slab['chunk_size'] < args.size: |
911 | small_slab = True |
912 | @@ -102,7 +103,9 @@ def determine_move(history, stats, diffs, memfree): |
913 | if slab['evicted_d'] > 0: |
914 | w[sid]['dirty'] = 1 |
915 | w[sid]['ev'] = 1 |
916 | - if slab['free_chunks'] > memfree[sid] and memfree[sid] > 0: |
917 | + if slab['free_chunks'] > memfree[sid]: |
918 | + free_enough = True |
919 | + if memfree[sid] > 0 and slab['free_chunks'] > (memfree[sid] * 2): |
920 | w[sid]['excess_free'] = 1 |
921 | w[sid]['age'] = slab['age'] |
922 | age = window_check(history, sid, 'age') / len(history['w']) |
923 | @@ -121,24 +124,25 @@ def determine_move(history, stats, diffs, memfree): |
924 | too_free = True |
925 | |
926 | # are we the oldest slab class? (and a valid target) |
927 | + # don't consider for young if we've recently given it unused memory |
928 | if small_slab == False: |
929 | if age > oldest[1] and slab['total_pages'] > MIN_PAGES_FOR_SOURCE: |
930 | oldest = (sid, age) |
931 | if age < youngest[1] and slab['total_pages'] > 0 \ |
932 | - and window_check(history, sid, 'excess_free') < len(history['w']): |
933 | + and window_check(history, sid, 'excess_free') < len(history['w']) \ |
934 | + and not (window_check(history, sid, 'relaxed') and free_enough): |
935 | youngest = (sid, age) |
936 | |
937 | |
938 | if w.get('slab_pool_high') and youngest[0] != -1: |
939 | # if global pool is too high, feed youngest large class. |
940 | - # decision = (0, youngest[0]) |
941 | - # No current way to assign directly from global to a class. |
942 | - # minimize the free chunks limiter and re-check. |
943 | - decision = (youngest[0], -2) |
944 | + if slab['free_chunks'] <= memfree[youngest[0]]: |
945 | + decision = (0, youngest[0]) |
946 | + w[youngest[0]]['relaxed'] = 1 |
947 | elif too_free == False and pool_low and oldest[0] != -1: |
948 | # if pool is too low, take from oldest large class. |
949 | if args.verbose: |
950 | - print("oldest: [class: {}] [age: {}]".format(int(oldest[0]), oldest[1])) |
951 | + print("oldest: [class: {}] [age: {:.2f}]".format(int(oldest[0]), oldest[1])) |
952 | decision = (oldest[0], 0) |
953 | elif too_free == False and youngest[0] != -1 and oldest[0] != -1 and youngest[0] != oldest[0]: |
954 | # youngest is outside of the tolerance ratio, move a page around. |
955 | @@ -148,8 +152,10 @@ def determine_move(history, stats, diffs, memfree): |
956 | |
957 | slab = diffs[youngest[0]] |
958 | #print("F:{} L:{} Y:{} R:{}".format(slab['free_chunks'], memfree[youngest[0]], int(youngest[1]), int(oldest[1] * args.ratio))) |
959 | - if (slab['free_chunks'] <= memfree[youngest[0]]) and (youngest[1] < oldest[1] * args.ratio): |
960 | - decision = (oldest[0], youngest[0]) |
961 | + if youngest[1] < oldest[1] * args.ratio: |
962 | + w[youngest[0]]['relaxed'] = 1 |
963 | + if slab['free_chunks'] <= memfree[youngest[0]]: |
964 | + decision = (0, youngest[0]) |
965 | |
966 | if (len(history['w']) >= args.window): |
967 | return decision |
968 | @@ -289,27 +295,16 @@ while True: |
969 | (diffs, totals) = diff_stats(stats_pre, stats_post) |
970 | #if args.verbose: |
971 | # show_detail(diffs, totals) |
972 | - if (last_memfree_check < time() - 60) and totals.get('total_pages'): |
973 | - memfree = memfree_check(s, diffs, totals) |
974 | - last_memfree_check = time() |
975 | - decision = (-1, -1) |
976 | if int(stats['evictions']) > 0: |
977 | + if (last_memfree_check < time() - 60) and totals.get('total_pages'): |
978 | + memfree = memfree_check(s, diffs, totals) |
979 | + last_memfree_check = time() |
980 | + decision = (-1, -1) |
981 | decision = determine_move(history, stats, diffs, memfree) |
982 | if int(decision[0]) > 0 and int(decision[1]) >= 0: |
983 | print("moving page from, to:", decision) |
984 | if args.automove: |
985 | run_move(s, decision) |
986 | - elif int(decision[0]) > 0 and decision[1] == -2: |
987 | - # we've been told to zero out the memory limit. |
988 | - # let the memfree check run "soon". |
989 | - # this (purposefully) can slowly remove limits on |
990 | - # multiple classes if global pool stays too high. |
991 | - last_memfree_check = time() - 58 |
992 | - s.write("extstore free_memchunks {} {}\r\n".format(decision[0], 0)) |
993 | - s.readline() |
994 | - if args.verbose: |
995 | - print("relaxing free:", decision[0]) |
996 | - |
997 | |
998 | # Minimize sleeping if we just moved a page to global pool. |
999 | # Improves responsiveness during flushes/quick changes. |
1000 | diff --git a/scripts/memcached.service b/scripts/memcached.service |
1001 | index 2bfbb67..88a4b8a 100644 |
1002 | --- a/scripts/memcached.service |
1003 | +++ b/scripts/memcached.service |
1004 | @@ -1,10 +1,10 @@ |
1005 | # It's not recommended to modify this file in-place, because it will be |
1006 | # overwritten during upgrades. If you want to customize, the best |
1007 | # way is to use the "systemctl edit" command to create an override unit. |
1008 | - |
1009 | +# |
1010 | # For example, to pass additional options, create an override unit |
1011 | # (as is done by systemctl edit) and enter the following: |
1012 | - |
1013 | +# |
1014 | # [Service] |
1015 | # Environment=OPTIONS="-l 127.0.0.1,::1" |
1016 | |
1017 | @@ -17,56 +17,68 @@ After=network.target |
1018 | EnvironmentFile=/etc/sysconfig/memcached |
1019 | ExecStart=/usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS |
1020 | |
1021 | -# Set up a new file system namespace and mounts private /tmp and /var/tmp directories |
1022 | -# so this service cannot access the global directories and other processes cannot |
1023 | -# access this service's directories. |
1024 | +# Set up a new file system namespace and mounts private /tmp and /var/tmp |
1025 | +# directories so this service cannot access the global directories and |
1026 | +# other processes cannot access this service's directories. |
1027 | PrivateTmp=true |
1028 | |
1029 | -# Mounts the /usr, /boot, and /etc directories read-only for processes invoked by this unit. |
1030 | +# Mounts the /usr, /boot, and /etc directories read-only for processes |
1031 | +# invoked by this unit. |
1032 | ProtectSystem=full |
1033 | |
1034 | -# Ensures that the service process and all its children can never gain new privileges |
1035 | +# Ensures that the service process and all its children can never gain new |
1036 | +# privileges |
1037 | NoNewPrivileges=true |
1038 | |
1039 | -# Sets up a new /dev namespace for the executed processes and only adds API pseudo devices |
1040 | -# such as /dev/null, /dev/zero or /dev/random (as well as the pseudo TTY subsystem) to it, |
1041 | -# but no physical devices such as /dev/sda. |
1042 | +# Sets up a new /dev namespace for the executed processes and only adds API |
1043 | +# pseudo devices such as /dev/null, /dev/zero or /dev/random (as well as |
1044 | +# the pseudo TTY subsystem) to it, but no physical devices such as /dev/sda. |
1045 | PrivateDevices=true |
1046 | |
1047 | # Required for dropping privileges and running as a different user |
1048 | CapabilityBoundingSet=CAP_SETGID CAP_SETUID CAP_SYS_RESOURCE |
1049 | |
1050 | -# Attempts to create memory mappings that are writable and executable at the same time, |
1051 | -# or to change existing memory mappings to become executable are prohibited. |
1052 | -MemoryDenyWriteExecute=true |
1053 | +# Restricts the set of socket address families accessible to the processes |
1054 | +# of this unit. Protects against vulnerabilities such as CVE-2016-8655 |
1055 | +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
1056 | |
1057 | -# Explicit module loading will be denied. This allows to turn off module load and unload |
1058 | -# operations on modular kernels. It is recommended to turn this on for most services that |
1059 | -# do not need special file systems or extra kernel modules to work. |
1060 | -ProtectKernelModules=true |
1061 | |
1062 | -# Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, /proc/latency_stats, |
1063 | -# /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq will be made read-only to all processes |
1064 | -# of the unit. Usually, tunable kernel variables should only be written at boot-time, with the |
1065 | -# sysctl.d(5) mechanism. Almost no services need to write to these at runtime; it is hence |
1066 | -# recommended to turn this on for most services. |
1067 | -ProtectKernelTunables=true |
1068 | +# Some security features are not in the older versions of systemd used by |
1069 | +# e.g. RHEL7/CentOS 7. The below settings are automatically edited at package |
1070 | +# build time to uncomment them if the target platform supports them. |
1071 | |
1072 | -# The Linux Control Groups (cgroups(7)) hierarchies accessible through /sys/fs/cgroup will be |
1073 | -# made read-only to all processes of the unit. Except for container managers no services should |
1074 | -# require write access to the control groups hierarchies; it is hence recommended to turn this on |
1075 | -# for most services |
1076 | -ProtectControlGroups=true |
1077 | +# Attempts to create memory mappings that are writable and executable at |
1078 | +# the same time, or to change existing memory mappings to become executable |
1079 | +# are prohibited. |
1080 | +##safer##MemoryDenyWriteExecute=true |
1081 | |
1082 | -# Any attempts to enable realtime scheduling in a process of the unit are refused. |
1083 | -RestrictRealtime=true |
1084 | +# Explicit module loading will be denied. This allows to turn off module |
1085 | +# load and unload operations on modular kernels. It is recommended to turn |
1086 | +# this on for most services that do not need special file systems or extra |
1087 | +# kernel modules to work. |
1088 | +##safer##ProtectKernelModules=true |
1089 | |
1090 | -# Restricts the set of socket address families accessible to the processes of this unit. |
1091 | -# Protects against vulnerabilities such as CVE-2016-8655 |
1092 | -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
1093 | +# Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, |
1094 | +# /proc/latency_stats, /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq |
1095 | +# will be made read-only to all processes of the unit. Usually, tunable |
1096 | +# kernel variables should only be written at boot-time, with the sysctl.d(5) |
1097 | +# mechanism. Almost no services need to write to these at runtime; it is hence |
1098 | +# recommended to turn this on for most services. |
1099 | +##safer##ProtectKernelTunables=true |
1100 | + |
1101 | +# The Linux Control Groups (cgroups(7)) hierarchies accessible through |
1102 | +# /sys/fs/cgroup will be made read-only to all processes of the unit. |
1103 | +# Except for container managers no services should require write access |
1104 | +# to the control groups hierarchies; it is hence recommended to turn this |
1105 | +# on for most services |
1106 | +##safer##ProtectControlGroups=true |
1107 | + |
1108 | +# Any attempts to enable realtime scheduling in a process of the unit are |
1109 | +# refused. |
1110 | +##safer##RestrictRealtime=true |
1111 | |
1112 | # Takes away the ability to create or manage any kind of namespace |
1113 | -RestrictNamespaces=true |
1114 | +##safer##RestrictNamespaces=true |
1115 | |
1116 | [Install] |
1117 | WantedBy=multi-user.target |
1118 | diff --git a/scripts/memcached.sysconfig b/scripts/memcached.sysconfig |
1119 | new file mode 100644 |
1120 | index 0000000..bd46483 |
1121 | --- /dev/null |
1122 | +++ b/scripts/memcached.sysconfig |
1123 | @@ -0,0 +1,10 @@ |
1124 | +# These defaults will be used by every memcached instance, unless overridden |
1125 | +# by values in /etc/sysconfig/memcached.<port> |
1126 | +USER="nobody" |
1127 | +MAXCONN="1024" |
1128 | +CACHESIZE="64" |
1129 | +OPTIONS="" |
1130 | + |
1131 | +# The PORT variable will only be used by memcached.service, not by |
1132 | +# memcached@xxxxx services, which will use the xxxxx |
1133 | +PORT="11211" |
1134 | diff --git a/scripts/memcached@.service b/scripts/memcached@.service |
1135 | new file mode 100644 |
1136 | index 0000000..4e9f1d7 |
1137 | --- /dev/null |
1138 | +++ b/scripts/memcached@.service |
1139 | @@ -0,0 +1,89 @@ |
1140 | +# It's not recommended to modify this file in-place, because it will be |
1141 | +# overwritten during upgrades. If you want to customize, the best |
1142 | +# way is to use the "systemctl edit" command to create an override unit. |
1143 | +# |
1144 | +# For example, to pass additional options, create an override unit |
1145 | +# (as is done by systemctl edit) and enter the following: |
1146 | +# |
1147 | +# [Service] |
1148 | +# Environment=OPTIONS="-l 127.0.0.1,::1" |
1149 | +# |
1150 | +# To use the "instanced" version of this, just start 'memcached@11211' or |
1151 | +# whatever port you'd like. If /etc/sysconfig/memcached.<port> exists, it |
1152 | +# will be read first, so you can set different parameters for a given |
1153 | +# instance. |
1154 | + |
1155 | +[Unit] |
1156 | +Description=memcached daemon |
1157 | +After=network.target |
1158 | + |
1159 | +[Service] |
1160 | +EnvironmentFile=/etc/sysconfig/memcached |
1161 | +EnvironmentFile=-/etc/sysconfig/memcached.%i |
1162 | +ExecStart=/usr/bin/memcached -p %i -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS |
1163 | + |
1164 | +# Set up a new file system namespace and mounts private /tmp and /var/tmp |
1165 | +# directories so this service cannot access the global directories and |
1166 | +# other processes cannot access this service's directories. |
1167 | +PrivateTmp=true |
1168 | + |
1169 | +# Mounts the /usr, /boot, and /etc directories read-only for processes |
1170 | +# invoked by this unit. |
1171 | +ProtectSystem=full |
1172 | + |
1173 | +# Ensures that the service process and all its children can never gain new |
1174 | +# privileges |
1175 | +NoNewPrivileges=true |
1176 | + |
1177 | +# Sets up a new /dev namespace for the executed processes and only adds API |
1178 | +# pseudo devices such as /dev/null, /dev/zero or /dev/random (as well as |
1179 | +# the pseudo TTY subsystem) to it, but no physical devices such as /dev/sda. |
1180 | +PrivateDevices=true |
1181 | + |
1182 | +# Required for dropping privileges and running as a different user |
1183 | +CapabilityBoundingSet=CAP_SETGID CAP_SETUID CAP_SYS_RESOURCE |
1184 | + |
1185 | +# Restricts the set of socket address families accessible to the processes |
1186 | +# of this unit. Protects against vulnerabilities such as CVE-2016-8655 |
1187 | +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
1188 | + |
1189 | + |
1190 | +# Some security features are not in the older versions of systemd used by |
1191 | +# e.g. RHEL7/CentOS 7. The below settings are automatically edited at package |
1192 | +# build time to uncomment them if the target platform supports them. |
1193 | + |
1194 | +# Attempts to create memory mappings that are writable and executable at |
1195 | +# the same time, or to change existing memory mappings to become executable |
1196 | +# are prohibited. |
1197 | +##safer##MemoryDenyWriteExecute=true |
1198 | + |
1199 | +# Explicit module loading will be denied. This allows to turn off module |
1200 | +# load and unload operations on modular kernels. It is recommended to turn |
1201 | +# this on for most services that do not need special file systems or extra |
1202 | +# kernel modules to work. |
1203 | +##safer##ProtectKernelModules=true |
1204 | + |
1205 | +# Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, |
1206 | +# /proc/latency_stats, /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq |
1207 | +# will be made read-only to all processes of the unit. Usually, tunable |
1208 | +# kernel variables should only be written at boot-time, with the sysctl.d(5) |
1209 | +# mechanism. Almost no services need to write to these at runtime; it is hence |
1210 | +# recommended to turn this on for most services. |
1211 | +##safer##ProtectKernelTunables=true |
1212 | + |
1213 | +# The Linux Control Groups (cgroups(7)) hierarchies accessible through |
1214 | +# /sys/fs/cgroup will be made read-only to all processes of the unit. |
1215 | +# Except for container managers no services should require write access |
1216 | +# to the control groups hierarchies; it is hence recommended to turn this |
1217 | +# on for most services |
1218 | +##safer##ProtectControlGroups=true |
1219 | + |
1220 | +# Any attempts to enable realtime scheduling in a process of the unit are |
1221 | +# refused. |
1222 | +##safer##RestrictRealtime=true |
1223 | + |
1224 | +# Takes away the ability to create or manage any kind of namespace |
1225 | +##safer##RestrictNamespaces=true |
1226 | + |
1227 | +[Install] |
1228 | +WantedBy=multi-user.target |
1229 | diff --git a/slab_automove_extstore.c b/slab_automove_extstore.c |
1230 | index d9b62b9..a3012d7 100644 |
1231 | --- a/slab_automove_extstore.c |
1232 | +++ b/slab_automove_extstore.c |
1233 | @@ -20,6 +20,7 @@ struct window_data { |
1234 | uint64_t dirty; |
1235 | uint64_t evicted; |
1236 | unsigned int excess_free; |
1237 | + unsigned int relaxed; |
1238 | }; |
1239 | |
1240 | struct window_global { |
1241 | @@ -90,6 +91,7 @@ static void window_sum(struct window_data *wd, struct window_data *w, |
1242 | w->dirty += d->dirty; |
1243 | w->evicted += d->evicted; |
1244 | w->excess_free += d->excess_free; |
1245 | + w->relaxed += d->relaxed; |
1246 | } |
1247 | } |
1248 | |
1249 | @@ -121,6 +123,15 @@ static void global_pool_check(slab_automove *a) { |
1250 | } |
1251 | } |
1252 | |
1253 | +/* A percentage of memory is configured to be held "free" as buffers for the |
1254 | + * external storage system. |
1255 | + * % of global memory is desired in the global page pool |
1256 | + * each slab class has a % of free chunks desired based on how much memory is |
1257 | + * currently in the class. This allows time for extstore to flush data when |
1258 | + * spikes or waves of set data arrive. |
1259 | + * The global page pool reserve acts as a secondary buffer for any slab class, |
1260 | + * which helps absorb shifts in which class is active. |
1261 | + */ |
1262 | static void memcheck(slab_automove *a) { |
1263 | unsigned int total_pages = 0; |
1264 | if (current_time < a->last_memcheck_run + MEMCHECK_PERIOD) |
1265 | @@ -143,6 +154,11 @@ static void memcheck(slab_automove *a) { |
1266 | a->free_mem[0] = total_pages * a->free_ratio; |
1267 | } |
1268 | |
1269 | +static struct window_data *get_window_data(slab_automove *a, int class) { |
1270 | + int w_offset = class * a->window_size; |
1271 | + return &a->window_data[w_offset + (a->window_cur % a->window_size)]; |
1272 | +} |
1273 | + |
1274 | void slab_automove_extstore_run(void *arg, int *src, int *dst) { |
1275 | slab_automove *a = (slab_automove *)arg; |
1276 | int n; |
1277 | @@ -170,15 +186,16 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) { |
1278 | for (n = POWER_SMALLEST; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) { |
1279 | bool small_slab = a->sam_before[n].chunk_size < a->item_size |
1280 | ? true : false; |
1281 | - int w_offset = n * a->window_size; |
1282 | - struct window_data *wd = &a->window_data[w_offset + (a->window_cur % a->window_size)]; |
1283 | + bool free_enough = false; |
1284 | + struct window_data *wd = get_window_data(a, n); |
1285 | // summarize the window-up-to-now. |
1286 | memset(&w_sum, 0, sizeof(struct window_data)); |
1287 | + int w_offset = n * a->window_size; |
1288 | window_sum(&a->window_data[w_offset], &w_sum, a->window_size); |
1289 | memset(wd, 0, sizeof(struct window_data)); |
1290 | |
1291 | - // if page delta, or evicted delta, mark window dirty |
1292 | - // (or outofmemory) |
1293 | + // if page delta, oom, or evicted delta, mark window dirty |
1294 | + // classes marked dirty cannot donate memory back to global pool. |
1295 | if (a->iam_after[n].evicted - a->iam_before[n].evicted > 0 || |
1296 | a->iam_after[n].outofmemory - a->iam_before[n].outofmemory > 0) { |
1297 | wd->evicted = 1; |
1298 | @@ -188,7 +205,14 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) { |
1299 | wd->dirty = 1; |
1300 | } |
1301 | // Mark excess free if we're over the free mem limit for too long. |
1302 | - if (a->sam_after[n].free_chunks > a->free_mem[n] && a->free_mem[n] > 0) { |
1303 | + // "free_enough" means it is either wobbling, recently received a new |
1304 | + // page of memory, or the crawler is freeing memory. |
1305 | + if (a->sam_after[n].free_chunks > a->free_mem[n]) { |
1306 | + free_enough = true; |
1307 | + } |
1308 | + // double the free requirements means we may have memory we can |
1309 | + // reclaim to global, if it stays this way for the whole window. |
1310 | + if (a->sam_after[n].free_chunks > (a->free_mem[n] * 2) && a->free_mem[n] > 0) { |
1311 | wd->excess_free = 1; |
1312 | } |
1313 | |
1314 | @@ -198,7 +222,9 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) { |
1315 | // grab age as average of window total |
1316 | uint64_t age = w_sum.age / a->window_size; |
1317 | |
1318 | - // if > N free chunks and not dirty, make decision. |
1319 | + // if > N free chunks and not dirty, reclaim memory |
1320 | + // small slab classes aren't age balanced and rely more on global |
1321 | + // pool. reclaim them more aggressively. |
1322 | if (a->sam_after[n].free_chunks > a->sam_after[n].chunks_per_page * MIN_PAGES_FOR_RECLAIM |
1323 | && w_sum.dirty == 0) { |
1324 | if (small_slab) { |
1325 | @@ -214,19 +240,22 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) { |
1326 | } |
1327 | } |
1328 | |
1329 | - // if oldest and have enough pages, is oldest |
1330 | - if (!small_slab |
1331 | - && age > oldest_age |
1332 | - && a->sam_after[n].total_pages > MIN_PAGES_FOR_SOURCE) { |
1333 | - oldest = n; |
1334 | - oldest_age = age; |
1335 | - } |
1336 | + if (!small_slab) { |
1337 | + // if oldest and have enough pages, is oldest |
1338 | + if (age > oldest_age |
1339 | + && a->sam_after[n].total_pages > MIN_PAGES_FOR_SOURCE) { |
1340 | + oldest = n; |
1341 | + oldest_age = age; |
1342 | + } |
1343 | |
1344 | - // don't count as youngest if it hasn't been using new chunks. |
1345 | - if (!small_slab && age < youngest_age && a->sam_after[n].total_pages != 0 |
1346 | - && w_sum.excess_free < a->window_size) { |
1347 | - youngest = n; |
1348 | - youngest_age = age; |
1349 | + // don't count as youngest if it hasn't been using new chunks. |
1350 | + // (if it was relaxed recently, and is currently "free enough") |
1351 | + if (age < youngest_age && a->sam_after[n].total_pages != 0 |
1352 | + && w_sum.excess_free < a->window_size |
1353 | + && !(w_sum.relaxed && free_enough)) { |
1354 | + youngest = n; |
1355 | + youngest_age = age; |
1356 | + } |
1357 | } |
1358 | } |
1359 | |
1360 | @@ -239,28 +268,29 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) { |
1361 | return; |
1362 | |
1363 | if (wg_sum.pool_high >= a->window_size && !wg_sum.pool_low && youngest != -1) { |
1364 | - /**src = 0; |
1365 | - *dst = youngest;*/ |
1366 | - /* TODO: No current way to directly assign page from 0 to elsewhere. |
1367 | - * Do a hack by setting the youngest's free mem limiter to |
1368 | - * zero and re-running memcheck in the next second. |
1369 | - * If set rates are very high and the pool is too low, this can bottom |
1370 | - * out... |
1371 | - */ |
1372 | - // schedule a memcheck run for "soon" to keep the limit zeroed out |
1373 | - // while the pool stays too high. This will also allow multiple |
1374 | - // classes to zero out over time. |
1375 | - a->last_memcheck_run = current_time - (MEMCHECK_PERIOD - 2); |
1376 | - a->settings->ext_free_memchunks[youngest] = 0; |
1377 | + if (a->sam_after[youngest].free_chunks <= a->free_mem[youngest]) { |
1378 | + *src = 0; |
1379 | + *dst = youngest; |
1380 | + } |
1381 | + struct window_data *wd = get_window_data(a, youngest); |
1382 | + // "relaxing" here and below allows us to skip classes which will |
1383 | + // never grow or are growing slowly, more quickly finding other |
1384 | + // classes which violate the age ratio. |
1385 | + wd->relaxed = 1; |
1386 | } else if (!too_free && wg_sum.pool_low && oldest != -1) { |
1387 | *src = oldest; |
1388 | *dst = 0; |
1389 | } else if (!too_free && youngest != -1 && oldest != -1 && youngest != oldest) { |
1390 | // if we have a youngest and oldest, and oldest is outside the ratio. |
1391 | - if (a->sam_after[youngest].free_chunks <= a->free_mem[youngest] |
1392 | - && youngest_age < ((double)oldest_age * a->max_age_ratio)) { |
1393 | - *src = oldest; |
1394 | - *dst = youngest; |
1395 | + if (youngest_age < ((double)oldest_age * a->max_age_ratio)) { |
1396 | + struct window_data *wd = get_window_data(a, youngest); |
1397 | + wd->relaxed = 1; |
1398 | + // only actually assign more memory if it's absorbed what it has |
1399 | + if (a->sam_after[youngest].free_chunks <= a->free_mem[youngest]) { |
1400 | + *src = 0; |
1401 | + *dst = youngest; |
1402 | + |
1403 | + } |
1404 | } |
1405 | } |
1406 | return; |
1407 | diff --git a/slabs.c b/slabs.c |
1408 | index cedb1e8..200d575 100644 |
1409 | --- a/slabs.c |
1410 | +++ b/slabs.c |
1411 | @@ -678,7 +678,7 @@ static int slab_rebalance_start(void) { |
1412 | |
1413 | pthread_mutex_lock(&slabs_lock); |
1414 | |
1415 | - if (slab_rebal.s_clsid < POWER_SMALLEST || |
1416 | + if (slab_rebal.s_clsid < SLAB_GLOBAL_PAGE_POOL || |
1417 | slab_rebal.s_clsid > power_largest || |
1418 | slab_rebal.d_clsid < SLAB_GLOBAL_PAGE_POOL || |
1419 | slab_rebal.d_clsid > power_largest || |
1420 | @@ -707,8 +707,11 @@ static int slab_rebalance_start(void) { |
1421 | (s_cls->size * s_cls->perslab); |
1422 | slab_rebal.slab_pos = slab_rebal.slab_start; |
1423 | slab_rebal.done = 0; |
1424 | + // Don't need to do chunk move work if page is in global pool. |
1425 | + if (slab_rebal.s_clsid == SLAB_GLOBAL_PAGE_POOL) { |
1426 | + slab_rebal.done = 1; |
1427 | + } |
1428 | |
1429 | - /* Also tells do_item_get to search for items in this slab */ |
1430 | slab_rebalance_signal = 2; |
1431 | |
1432 | if (settings.verbose > 1) { |
1433 | @@ -862,7 +865,9 @@ static int slab_rebalance_move(void) { |
1434 | // Only safe to hold slabs lock because refcount |
1435 | // can't drop to 0 until we release item lock. |
1436 | STORAGE_delete(storage, it); |
1437 | + pthread_mutex_unlock(&slabs_lock); |
1438 | do_item_unlink(it, hv); |
1439 | + pthread_mutex_lock(&slabs_lock); |
1440 | } |
1441 | status = MOVE_BUSY; |
1442 | } else { |
1443 | @@ -1187,7 +1192,7 @@ static enum reassign_result_type do_slabs_reassign(int src, int dst) { |
1444 | /* TODO: If we end up back at -1, return a new error type */ |
1445 | } |
1446 | |
1447 | - if (src < POWER_SMALLEST || src > power_largest || |
1448 | + if (src < SLAB_GLOBAL_PAGE_POOL || src > power_largest || |
1449 | dst < SLAB_GLOBAL_PAGE_POOL || dst > power_largest) |
1450 | return REASSIGN_BADCLASS; |
1451 | |
1452 | diff --git a/t/issue_67.t b/t/issue_67.t |
1453 | index 69c869f..b2d374f 100644 |
1454 | --- a/t/issue_67.t |
1455 | +++ b/t/issue_67.t |
1456 | @@ -77,11 +77,11 @@ sub when { |
1457 | |
1458 | # Disabling the defaults since it conflicts with a running instance. |
1459 | # when('no arguments', '', 11211, 11211); |
1460 | -when('specifying tcp port', '-p 11212', 11212, 11212); |
1461 | +when('specifying tcp port', '-p 11212', 11212, -1); |
1462 | when('specifying udp port', '-U 11222', 11222, 11222); |
1463 | when('specifying tcp ephemeral port', '-p -1', 0, 0); |
1464 | when('specifying udp ephemeral port', '-U -1', 0, 0); |
1465 | -when('tcp port disabled', '-p 0', -1, 11211); |
1466 | +when('tcp port disabled', '-p 0', -1, -1); |
1467 | when('udp port disabled', '-U 0', 11211, -1); |
1468 | when('specifying tcp and udp ports', '-p 11232 -U 11233', 11232, 11233); |
1469 | when('specifying tcp and disabling udp', '-p 11242 -U 0', 11242, -1); |
1470 | diff --git a/thread.c b/thread.c |
1471 | index df05d5f..d17a387 100644 |
1472 | --- a/thread.c |
1473 | +++ b/thread.c |
1474 | @@ -309,7 +309,16 @@ void accept_new_conns(const bool do_accept) { |
1475 | * Set up a thread's information. |
1476 | */ |
1477 | static void setup_thread(LIBEVENT_THREAD *me) { |
1478 | +#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= 0x02000101 |
1479 | + struct event_config *ev_config; |
1480 | + ev_config = event_config_new(); |
1481 | + event_config_set_flag(ev_config, EVENT_BASE_FLAG_NOLOCK); |
1482 | + me->base = event_base_new_with_config(ev_config); |
1483 | + event_config_free(ev_config); |
1484 | +#else |
1485 | me->base = event_init(); |
1486 | +#endif |
1487 | + |
1488 | if (! me->base) { |
1489 | fprintf(stderr, "Can't allocate event base\n"); |
1490 | exit(1); |
1491 | @@ -374,6 +383,8 @@ static void *worker_libevent(void *arg) { |
1492 | register_thread_initialized(); |
1493 | |
1494 | event_base_loop(me->base, 0); |
1495 | + |
1496 | + event_base_free(me->base); |
1497 | return NULL; |
1498 | } |
1499 | |
1500 | diff --git a/timedrun.c b/timedrun.c |
1501 | index 6d7afb3..6888453 100644 |
1502 | --- a/timedrun.c |
1503 | +++ b/timedrun.c |
1504 | @@ -17,7 +17,7 @@ static void caught_signal(int which) |
1505 | static int wait_for_process(pid_t pid) |
1506 | { |
1507 | int rv = EX_SOFTWARE; |
1508 | - int stats = 0; |
1509 | + int status = 0; |
1510 | int i = 0; |
1511 | struct sigaction sig_handler; |
1512 | |
1513 | @@ -32,12 +32,12 @@ static int wait_for_process(pid_t pid) |
1514 | |
1515 | /* Loop forever waiting for the process to quit */ |
1516 | for (i = 0; ;i++) { |
1517 | - pid_t p = waitpid(pid, &stats, 0); |
1518 | + pid_t p = waitpid(pid, &status, 0); |
1519 | if (p == pid) { |
1520 | /* child exited. Let's get out of here */ |
1521 | - rv = WIFEXITED(stats) ? |
1522 | - WEXITSTATUS(stats) : |
1523 | - (0x80 | WTERMSIG(stats)); |
1524 | + rv = WIFEXITED(status) ? |
1525 | + WEXITSTATUS(status) : |
1526 | + (0x80 | WTERMSIG(status)); |
1527 | break; |
1528 | } else { |
1529 | int sig = 0; |
1530 | diff --git a/trace.h b/trace.h |
1531 | index dc792a0..94e585d 100644 |
1532 | --- a/trace.h |
1533 | +++ b/trace.h |
1534 | @@ -4,11 +4,11 @@ |
1535 | #ifdef ENABLE_DTRACE |
1536 | #include "memcached_dtrace.h" |
1537 | #else |
1538 | -#define MEMCACHED_ASSOC_DELETE(arg0, arg1, arg2) |
1539 | +#define MEMCACHED_ASSOC_DELETE(arg0, arg1) |
1540 | #define MEMCACHED_ASSOC_DELETE_ENABLED() (0) |
1541 | #define MEMCACHED_ASSOC_FIND(arg0, arg1, arg2) |
1542 | #define MEMCACHED_ASSOC_FIND_ENABLED() (0) |
1543 | -#define MEMCACHED_ASSOC_INSERT(arg0, arg1, arg2) |
1544 | +#define MEMCACHED_ASSOC_INSERT(arg0, arg1) |
1545 | #define MEMCACHED_ASSOC_INSERT_ENABLED() (0) |
1546 | #define MEMCACHED_COMMAND_ADD(arg0, arg1, arg2, arg3, arg4) |
1547 | #define MEMCACHED_COMMAND_ADD_ENABLED() (0) |
1548 | diff --git a/version.m4 b/version.m4 |
1549 | index d763a82..4a02156 100644 |
1550 | --- a/version.m4 |
1551 | +++ b/version.m4 |
1552 | @@ -1 +1 @@ |
1553 | -m4_define([VERSION_NUMBER], [1.5.4]) |
1554 | +m4_define([VERSION_NUMBER], [1.5.6]) |
available in ppa:racb/ experimental