memcached-1.5.22/0000775000175000017500000000000013615454045010521 500000000000000memcached-1.5.22/m4/0000775000175000017500000000000013615454045011041 500000000000000memcached-1.5.22/m4/c99-backport.m40000664000175000017500000001210413250303500013410 00000000000000# AC_PROG_CC_C99 ([ACTION-IF-AVAILABLE], [ACTION-IF-UNAVAILABLE]) # ---------------------------------------------------------------- # If the C compiler is not in ISO C99 mode by default, try to add an # option to output variable CC to make it so. This macro tries # various options that select ISO C99 on some system or another. It # considers the compiler to be in ISO C99 mode if it handles _Bool, # // comments, flexible array members, inline, long long int, mixed # code and declarations, named initialization of structs, restrict, # va_copy, varargs macros, variable declarations in for loops and # variable length arrays. AC_DEFUN([AC_PROG_CC_C99], [AC_C_STD_TRY([c99], [[#include #include #include #include #include // Check varargs macros. These examples are taken from C99 6.10.3.5. #define debug(...) fprintf (stderr, __VA_ARGS__) #define showlist(...) puts (#__VA_ARGS__) #define report(test,...) ((test) ? puts (#test) : printf (__VA_ARGS__)) static void test_varargs_macros (void) { int x = 1234; int y = 5678; debug ("Flag"); debug ("X = %d\n", x); showlist (The first, second, and third items.); report (x>y, "x is %d but y is %d", x, y); } // Check long long types. #define BIG64 18446744073709551615ull #define BIG32 4294967295ul #define BIG_OK (BIG64 / BIG32 == 4294967297ull && BIG64 % BIG32 == 0) #if !BIG_OK your preprocessor is broken; #endif #if BIG_OK #else your preprocessor is broken; #endif static long long int bignum = -9223372036854775807LL; static unsigned long long int ubignum = BIG64; struct incomplete_array { int datasize; double data[]; }; struct named_init { int number; const wchar_t *name; double average; }; typedef const char *ccp; static inline int test_restrict (ccp restrict text) { // See if C++-style comments work. // Iterate through items via the restricted pointer. // Also check for declarations in for loops. for (unsigned int i = 0; *(text+i) != '\0'; ++i) continue; return 0; } // Check varargs and va_copy. static void test_varargs (const char *format, ...) { va_list args; va_start (args, format); va_list args_copy; va_copy (args_copy, args); const char *str; int number; float fnumber; while (*format) { switch (*format++) { case 's': // string str = va_arg (args_copy, const char *); break; case 'd': // int number = va_arg (args_copy, int); break; case 'f': // float fnumber = va_arg (args_copy, double); break; default: break; } } va_end (args_copy); va_end (args); } ]], [[ // Check bool. _Bool success = false; // Check restrict. if (test_restrict ("String literal") == 0) success = true; char *restrict newvar = "Another string"; // Check varargs. test_varargs ("s, d' f .", "string", 65, 34.234); test_varargs_macros (); // Check flexible array members. struct incomplete_array *ia = malloc (sizeof (struct incomplete_array) + (sizeof (double) * 10)); ia->datasize = 10; for (int i = 0; i < ia->datasize; ++i) ia->data[i] = i * 1.234; // Check named initializers. struct named_init ni = { .number = 34, .name = L"Test wide string", .average = 543.34343, }; ni.number = 58; int dynamic_array[ni.number]; dynamic_array[ni.number - 1] = 543; // work around unused variable warnings return (!success || bignum == 0LL || ubignum == 0uLL || newvar[0] == 'x' || dynamic_array[ni.number - 1] != 543); ]], dnl Try dnl GCC -std=gnu99 (unused restrictive modes: -std=c99 -std=iso9899:1999) dnl AIX -qlanglvl=extc99 (unused restrictive mode: -qlanglvl=stdc99) dnl Intel ICC -c99 dnl IRIX -c99 dnl Solaris (unused because it causes the compiler to assume C99 semantics for dnl library functions, and this is invalid before Solaris 10: -xc99) dnl Tru64 -c99 dnl with extended modes being tried first. [[-std=gnu99 -c99 -qlanglvl=extc99]], [$1], [$2])[]dnl ])# AC_PROG_CC_C99 # AC_C_STD_TRY(STANDARD, TEST-PROLOGUE, TEST-BODY, OPTION-LIST, # ACTION-IF-AVAILABLE, ACTION-IF-UNAVAILABLE) # -------------------------------------------------------------- # Check whether the C compiler accepts features of STANDARD (e.g `c89', `c99') # by trying to compile a program of TEST-PROLOGUE and TEST-BODY. If this fails, # try again with each compiler option in the space-separated OPTION-LIST; if one # helps, append it to CC. If eventually successful, run ACTION-IF-AVAILABLE, # else ACTION-IF-UNAVAILABLE. AC_DEFUN([AC_C_STD_TRY], [AC_MSG_CHECKING([for $CC option to accept ISO ]m4_translit($1, [c], [C])) AC_CACHE_VAL(ac_cv_prog_cc_$1, [ac_cv_prog_cc_$1=no ac_save_CC=$CC AC_LANG_CONFTEST([AC_LANG_PROGRAM([$2], [$3])]) for ac_arg in '' $4 do CC="$ac_save_CC $ac_arg" _AC_COMPILE_IFELSE([], [ac_cv_prog_cc_$1=$ac_arg]) test "x$ac_cv_prog_cc_$1" != "xno" && break done rm -f conftest.$ac_ext CC=$ac_save_CC ])# AC_CACHE_VAL case "x$ac_cv_prog_cc_$1" in x) AC_MSG_RESULT([none needed]) ;; xno) AC_MSG_RESULT([unsupported]) ;; *) CC="$CC $ac_cv_prog_cc_$1" AC_MSG_RESULT([$ac_cv_prog_cc_$1]) ;; esac AS_IF([test "x$ac_cv_prog_cc_$1" != xno], [$5], [$6]) ])# AC_C_STD_TRY memcached-1.5.22/assoc.c0000664000175000017500000002237013575022500011711 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ /* * Hash table * * The hash function used here is by Bob Jenkins, 1996: * * "By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. * You may use this code any way you wish, private, educational, * or commercial. It's free." * * The rest of the file is licensed under the BSD license. See LICENSE. */ #include "memcached.h" #include #include #include #include #include #include #include #include #include #include #include #include static pthread_cond_t maintenance_cond = PTHREAD_COND_INITIALIZER; static pthread_mutex_t maintenance_lock = PTHREAD_MUTEX_INITIALIZER; typedef uint32_t ub4; /* unsigned 4-byte quantities */ typedef unsigned char ub1; /* unsigned 1-byte quantities */ /* how many powers of 2's worth of buckets we use */ unsigned int hashpower = HASHPOWER_DEFAULT; #define hashsize(n) ((ub4)1<<(n)) #define hashmask(n) (hashsize(n)-1) /* Main hash table. This is where we look except during expansion. */ static item** primary_hashtable = 0; /* * Previous hash table. During expansion, we look here for keys that haven't * been moved over to the primary yet. */ static item** old_hashtable = 0; /* Flag: Are we in the middle of expanding now? */ static bool expanding = false; /* * During expansion we migrate values with bucket granularity; this is how * far we've gotten so far. Ranges from 0 .. hashsize(hashpower - 1) - 1. */ static unsigned int expand_bucket = 0; void assoc_init(const int hashtable_init) { if (hashtable_init) { hashpower = hashtable_init; } primary_hashtable = calloc(hashsize(hashpower), sizeof(void *)); if (! primary_hashtable) { fprintf(stderr, "Failed to init hashtable.\n"); exit(EXIT_FAILURE); } STATS_LOCK(); stats_state.hash_power_level = hashpower; stats_state.hash_bytes = hashsize(hashpower) * sizeof(void *); STATS_UNLOCK(); } item *assoc_find(const char *key, const size_t nkey, const uint32_t hv) { item *it; unsigned int oldbucket; if (expanding && (oldbucket = (hv & hashmask(hashpower - 1))) >= expand_bucket) { it = old_hashtable[oldbucket]; } else { it = primary_hashtable[hv & hashmask(hashpower)]; } item *ret = NULL; int depth = 0; while (it) { if ((nkey == it->nkey) && (memcmp(key, ITEM_key(it), nkey) == 0)) { ret = it; break; } it = it->h_next; ++depth; } MEMCACHED_ASSOC_FIND(key, nkey, depth); return ret; } /* returns the address of the item pointer before the key. if *item == 0, the item wasn't found */ static item** _hashitem_before (const char *key, const size_t nkey, const uint32_t hv) { item **pos; unsigned int oldbucket; if (expanding && (oldbucket = (hv & hashmask(hashpower - 1))) >= expand_bucket) { pos = &old_hashtable[oldbucket]; } else { pos = &primary_hashtable[hv & hashmask(hashpower)]; } while (*pos && ((nkey != (*pos)->nkey) || memcmp(key, ITEM_key(*pos), nkey))) { pos = &(*pos)->h_next; } return pos; } /* grows the hashtable to the next power of 2. */ static void assoc_expand(void) { old_hashtable = primary_hashtable; primary_hashtable = calloc(hashsize(hashpower + 1), sizeof(void *)); if (primary_hashtable) { if (settings.verbose > 1) fprintf(stderr, "Hash table expansion starting\n"); hashpower++; expanding = true; expand_bucket = 0; STATS_LOCK(); stats_state.hash_power_level = hashpower; stats_state.hash_bytes += hashsize(hashpower) * sizeof(void *); stats_state.hash_is_expanding = true; STATS_UNLOCK(); } else { primary_hashtable = old_hashtable; /* Bad news, but we can keep running. */ } } void assoc_start_expand(uint64_t curr_items) { if (pthread_mutex_trylock(&maintenance_lock) == 0) { if (curr_items > (hashsize(hashpower) * 3) / 2 && hashpower < HASHPOWER_MAX) { pthread_cond_signal(&maintenance_cond); } pthread_mutex_unlock(&maintenance_lock); } } /* Note: this isn't an assoc_update. The key must not already exist to call this */ int assoc_insert(item *it, const uint32_t hv) { unsigned int oldbucket; // assert(assoc_find(ITEM_key(it), it->nkey) == 0); /* shouldn't have duplicately named things defined */ if (expanding && (oldbucket = (hv & hashmask(hashpower - 1))) >= expand_bucket) { it->h_next = old_hashtable[oldbucket]; old_hashtable[oldbucket] = it; } else { it->h_next = primary_hashtable[hv & hashmask(hashpower)]; primary_hashtable[hv & hashmask(hashpower)] = it; } MEMCACHED_ASSOC_INSERT(ITEM_key(it), it->nkey); return 1; } void assoc_delete(const char *key, const size_t nkey, const uint32_t hv) { item **before = _hashitem_before(key, nkey, hv); if (*before) { item *nxt; /* The DTrace probe cannot be triggered as the last instruction * due to possible tail-optimization by the compiler */ MEMCACHED_ASSOC_DELETE(key, nkey); nxt = (*before)->h_next; (*before)->h_next = 0; /* probably pointless, but whatever. */ *before = nxt; return; } /* Note: we never actually get here. the callers don't delete things they can't find. */ assert(*before != 0); } static volatile int do_run_maintenance_thread = 1; #define DEFAULT_HASH_BULK_MOVE 1 int hash_bulk_move = DEFAULT_HASH_BULK_MOVE; static void *assoc_maintenance_thread(void *arg) { mutex_lock(&maintenance_lock); while (do_run_maintenance_thread) { int ii = 0; /* There is only one expansion thread, so no need to global lock. */ for (ii = 0; ii < hash_bulk_move && expanding; ++ii) { item *it, *next; unsigned int bucket; void *item_lock = NULL; /* bucket = hv & hashmask(hashpower) =>the bucket of hash table * is the lowest N bits of the hv, and the bucket of item_locks is * also the lowest M bits of hv, and N is greater than M. * So we can process expanding with only one item_lock. cool! */ if ((item_lock = item_trylock(expand_bucket))) { for (it = old_hashtable[expand_bucket]; NULL != it; it = next) { next = it->h_next; bucket = hash(ITEM_key(it), it->nkey) & hashmask(hashpower); it->h_next = primary_hashtable[bucket]; primary_hashtable[bucket] = it; } old_hashtable[expand_bucket] = NULL; expand_bucket++; if (expand_bucket == hashsize(hashpower - 1)) { expanding = false; free(old_hashtable); STATS_LOCK(); stats_state.hash_bytes -= hashsize(hashpower - 1) * sizeof(void *); stats_state.hash_is_expanding = false; STATS_UNLOCK(); if (settings.verbose > 1) fprintf(stderr, "Hash table expansion done\n"); } } else { usleep(10*1000); } if (item_lock) { item_trylock_unlock(item_lock); item_lock = NULL; } } if (!expanding) { /* We are done expanding.. just wait for next invocation */ pthread_cond_wait(&maintenance_cond, &maintenance_lock); /* assoc_expand() swaps out the hash table entirely, so we need * all threads to not hold any references related to the hash * table while this happens. * This is instead of a more complex, possibly slower algorithm to * allow dynamic hash table expansion without causing significant * wait times. */ if (do_run_maintenance_thread) { pause_threads(PAUSE_ALL_THREADS); assoc_expand(); pause_threads(RESUME_ALL_THREADS); } } } mutex_unlock(&maintenance_lock); return NULL; } static pthread_t maintenance_tid; int start_assoc_maintenance_thread() { int ret; char *env = getenv("MEMCACHED_HASH_BULK_MOVE"); if (env != NULL) { hash_bulk_move = atoi(env); if (hash_bulk_move == 0) { hash_bulk_move = DEFAULT_HASH_BULK_MOVE; } } if ((ret = pthread_create(&maintenance_tid, NULL, assoc_maintenance_thread, NULL)) != 0) { fprintf(stderr, "Can't create thread: %s\n", strerror(ret)); return -1; } return 0; } void stop_assoc_maintenance_thread() { mutex_lock(&maintenance_lock); do_run_maintenance_thread = 0; pthread_cond_signal(&maintenance_cond); mutex_unlock(&maintenance_lock); /* Wait for the maintenance thread to stop */ pthread_join(maintenance_tid, NULL); } memcached-1.5.22/crc32c.h0000664000175000017500000000026313553270304011665 00000000000000#ifndef CRC32C_H #define CRC32C_H typedef uint32_t (*crc_func)(uint32_t crc, const void *buf, size_t len); crc_func crc32c; void crc32c_init(void); #endif /* CRC32C_H */ memcached-1.5.22/depcomp0000755000175000017500000005601713256032133012014 00000000000000#! /bin/sh # depcomp - compile a program generating dependencies as side-effects scriptversion=2016-01-11.22; # UTC # Copyright (C) 1999-2017 Free Software Foundation, Inc. # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2, or (at your option) # any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. # Originally written by Alexandre Oliva . case $1 in '') echo "$0: No command. Try '$0 --help' for more information." 1>&2 exit 1; ;; -h | --h*) cat <<\EOF Usage: depcomp [--help] [--version] PROGRAM [ARGS] Run PROGRAMS ARGS to compile a file, generating dependencies as side-effects. Environment variables: depmode Dependency tracking mode. source Source file read by 'PROGRAMS ARGS'. object Object file output by 'PROGRAMS ARGS'. DEPDIR directory where to store dependencies. depfile Dependency file to output. tmpdepfile Temporary file to use when outputting dependencies. libtool Whether libtool is used (yes/no). Report bugs to . EOF exit $? ;; -v | --v*) echo "depcomp $scriptversion" exit $? ;; esac # Get the directory component of the given path, and save it in the # global variables '$dir'. Note that this directory component will # be either empty or ending with a '/' character. This is deliberate. set_dir_from () { case $1 in */*) dir=`echo "$1" | sed -e 's|/[^/]*$|/|'`;; *) dir=;; esac } # Get the suffix-stripped basename of the given path, and save it the # global variable '$base'. set_base_from () { base=`echo "$1" | sed -e 's|^.*/||' -e 's/\.[^.]*$//'` } # If no dependency file was actually created by the compiler invocation, # we still have to create a dummy depfile, to avoid errors with the # Makefile "include basename.Plo" scheme. make_dummy_depfile () { echo "#dummy" > "$depfile" } # Factor out some common post-processing of the generated depfile. # Requires the auxiliary global variable '$tmpdepfile' to be set. aix_post_process_depfile () { # If the compiler actually managed to produce a dependency file, # post-process it. if test -f "$tmpdepfile"; then # Each line is of the form 'foo.o: dependency.h'. # Do two passes, one to just change these to # $object: dependency.h # and one to simply output # dependency.h: # which is needed to avoid the deleted-header problem. { sed -e "s,^.*\.[$lower]*:,$object:," < "$tmpdepfile" sed -e "s,^.*\.[$lower]*:[$tab ]*,," -e 's,$,:,' < "$tmpdepfile" } > "$depfile" rm -f "$tmpdepfile" else make_dummy_depfile fi } # A tabulation character. tab=' ' # A newline character. nl=' ' # Character ranges might be problematic outside the C locale. # These definitions help. upper=ABCDEFGHIJKLMNOPQRSTUVWXYZ lower=abcdefghijklmnopqrstuvwxyz digits=0123456789 alpha=${upper}${lower} if test -z "$depmode" || test -z "$source" || test -z "$object"; then echo "depcomp: Variables source, object and depmode must be set" 1>&2 exit 1 fi # Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po. depfile=${depfile-`echo "$object" | sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`} tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`} rm -f "$tmpdepfile" # Avoid interferences from the environment. gccflag= dashmflag= # Some modes work just like other modes, but use different flags. We # parameterize here, but still list the modes in the big case below, # to make depend.m4 easier to write. Note that we *cannot* use a case # here, because this file can only contain one case statement. if test "$depmode" = hp; then # HP compiler uses -M and no extra arg. gccflag=-M depmode=gcc fi if test "$depmode" = dashXmstdout; then # This is just like dashmstdout with a different argument. dashmflag=-xM depmode=dashmstdout fi cygpath_u="cygpath -u -f -" if test "$depmode" = msvcmsys; then # This is just like msvisualcpp but w/o cygpath translation. # Just convert the backslash-escaped backslashes to single forward # slashes to satisfy depend.m4 cygpath_u='sed s,\\\\,/,g' depmode=msvisualcpp fi if test "$depmode" = msvc7msys; then # This is just like msvc7 but w/o cygpath translation. # Just convert the backslash-escaped backslashes to single forward # slashes to satisfy depend.m4 cygpath_u='sed s,\\\\,/,g' depmode=msvc7 fi if test "$depmode" = xlc; then # IBM C/C++ Compilers xlc/xlC can output gcc-like dependency information. gccflag=-qmakedep=gcc,-MF depmode=gcc fi case "$depmode" in gcc3) ## gcc 3 implements dependency tracking that does exactly what ## we want. Yay! Note: for some reason libtool 1.4 doesn't like ## it if -MD -MP comes after the -MF stuff. Hmm. ## Unfortunately, FreeBSD c89 acceptance of flags depends upon ## the command line argument order; so add the flags where they ## appear in depend2.am. Note that the slowdown incurred here ## affects only configure: in makefiles, %FASTDEP% shortcuts this. for arg do case $arg in -c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;; *) set fnord "$@" "$arg" ;; esac shift # fnord shift # $arg done "$@" stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi mv "$tmpdepfile" "$depfile" ;; gcc) ## Note that this doesn't just cater to obsosete pre-3.x GCC compilers. ## but also to in-use compilers like IMB xlc/xlC and the HP C compiler. ## (see the conditional assignment to $gccflag above). ## There are various ways to get dependency output from gcc. Here's ## why we pick this rather obscure method: ## - Don't want to use -MD because we'd like the dependencies to end ## up in a subdir. Having to rename by hand is ugly. ## (We might end up doing this anyway to support other compilers.) ## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like ## -MM, not -M (despite what the docs say). Also, it might not be ## supported by the other compilers which use the 'gcc' depmode. ## - Using -M directly means running the compiler twice (even worse ## than renaming). if test -z "$gccflag"; then gccflag=-MD, fi "$@" -Wp,"$gccflag$tmpdepfile" stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" echo "$object : \\" > "$depfile" # The second -e expression handles DOS-style file names with drive # letters. sed -e 's/^[^:]*: / /' \ -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile" ## This next piece of magic avoids the "deleted header file" problem. ## The problem is that when a header file which appears in a .P file ## is deleted, the dependency causes make to die (because there is ## typically no way to rebuild the header). We avoid this by adding ## dummy dependencies for each header file. Too bad gcc doesn't do ## this for us directly. ## Some versions of gcc put a space before the ':'. On the theory ## that the space means something, we add a space to the output as ## well. hp depmode also adds that space, but also prefixes the VPATH ## to the object. Take care to not repeat it in the output. ## Some versions of the HPUX 10.20 sed can't process this invocation ## correctly. Breaking it into two sed invocations is a workaround. tr ' ' "$nl" < "$tmpdepfile" \ | sed -e 's/^\\$//' -e '/^$/d' -e "s|.*$object$||" -e '/:$/d' \ | sed -e 's/$/ :/' >> "$depfile" rm -f "$tmpdepfile" ;; hp) # This case exists only to let depend.m4 do its work. It works by # looking at the text of this script. This case will never be run, # since it is checked for above. exit 1 ;; sgi) if test "$libtool" = yes; then "$@" "-Wp,-MDupdate,$tmpdepfile" else "$@" -MDupdate "$tmpdepfile" fi stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files echo "$object : \\" > "$depfile" # Clip off the initial element (the dependent). Don't try to be # clever and replace this with sed code, as IRIX sed won't handle # lines with more than a fixed number of characters (4096 in # IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines; # the IRIX cc adds comments like '#:fec' to the end of the # dependency line. tr ' ' "$nl" < "$tmpdepfile" \ | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' \ | tr "$nl" ' ' >> "$depfile" echo >> "$depfile" # The second pass generates a dummy entry for each header file. tr ' ' "$nl" < "$tmpdepfile" \ | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \ >> "$depfile" else make_dummy_depfile fi rm -f "$tmpdepfile" ;; xlc) # This case exists only to let depend.m4 do its work. It works by # looking at the text of this script. This case will never be run, # since it is checked for above. exit 1 ;; aix) # The C for AIX Compiler uses -M and outputs the dependencies # in a .u file. In older versions, this file always lives in the # current directory. Also, the AIX compiler puts '$object:' at the # start of each line; $object doesn't have directory information. # Version 6 uses the directory in both cases. set_dir_from "$object" set_base_from "$object" if test "$libtool" = yes; then tmpdepfile1=$dir$base.u tmpdepfile2=$base.u tmpdepfile3=$dir.libs/$base.u "$@" -Wc,-M else tmpdepfile1=$dir$base.u tmpdepfile2=$dir$base.u tmpdepfile3=$dir$base.u "$@" -M fi stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" exit $stat fi for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" do test -f "$tmpdepfile" && break done aix_post_process_depfile ;; tcc) # tcc (Tiny C Compiler) understand '-MD -MF file' since version 0.9.26 # FIXME: That version still under development at the moment of writing. # Make that this statement remains true also for stable, released # versions. # It will wrap lines (doesn't matter whether long or short) with a # trailing '\', as in: # # foo.o : \ # foo.c \ # foo.h \ # # It will put a trailing '\' even on the last line, and will use leading # spaces rather than leading tabs (at least since its commit 0394caf7 # "Emit spaces for -MD"). "$@" -MD -MF "$tmpdepfile" stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" # Each non-empty line is of the form 'foo.o : \' or ' dep.h \'. # We have to change lines of the first kind to '$object: \'. sed -e "s|.*:|$object :|" < "$tmpdepfile" > "$depfile" # And for each line of the second kind, we have to emit a 'dep.h:' # dummy dependency, to avoid the deleted-header problem. sed -n -e 's|^ *\(.*\) *\\$|\1:|p' < "$tmpdepfile" >> "$depfile" rm -f "$tmpdepfile" ;; ## The order of this option in the case statement is important, since the ## shell code in configure will try each of these formats in the order ## listed in this file. A plain '-MD' option would be understood by many ## compilers, so we must ensure this comes after the gcc and icc options. pgcc) # Portland's C compiler understands '-MD'. # Will always output deps to 'file.d' where file is the root name of the # source file under compilation, even if file resides in a subdirectory. # The object file name does not affect the name of the '.d' file. # pgcc 10.2 will output # foo.o: sub/foo.c sub/foo.h # and will wrap long lines using '\' : # foo.o: sub/foo.c ... \ # sub/foo.h ... \ # ... set_dir_from "$object" # Use the source, not the object, to determine the base name, since # that's sadly what pgcc will do too. set_base_from "$source" tmpdepfile=$base.d # For projects that build the same source file twice into different object # files, the pgcc approach of using the *source* file root name can cause # problems in parallel builds. Use a locking strategy to avoid stomping on # the same $tmpdepfile. lockdir=$base.d-lock trap " echo '$0: caught signal, cleaning up...' >&2 rmdir '$lockdir' exit 1 " 1 2 13 15 numtries=100 i=$numtries while test $i -gt 0; do # mkdir is a portable test-and-set. if mkdir "$lockdir" 2>/dev/null; then # This process acquired the lock. "$@" -MD stat=$? # Release the lock. rmdir "$lockdir" break else # If the lock is being held by a different process, wait # until the winning process is done or we timeout. while test -d "$lockdir" && test $i -gt 0; do sleep 1 i=`expr $i - 1` done fi i=`expr $i - 1` done trap - 1 2 13 15 if test $i -le 0; then echo "$0: failed to acquire lock after $numtries attempts" >&2 echo "$0: check lockdir '$lockdir'" >&2 exit 1 fi if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" # Each line is of the form `foo.o: dependent.h', # or `foo.o: dep1.h dep2.h \', or ` dep3.h dep4.h \'. # Do two passes, one to just change these to # `$object: dependent.h' and one to simply `dependent.h:'. sed "s,^[^:]*:,$object :," < "$tmpdepfile" > "$depfile" # Some versions of the HPUX 10.20 sed can't process this invocation # correctly. Breaking it into two sed invocations is a workaround. sed 's,^[^:]*: \(.*\)$,\1,;s/^\\$//;/^$/d;/:$/d' < "$tmpdepfile" \ | sed -e 's/$/ :/' >> "$depfile" rm -f "$tmpdepfile" ;; hp2) # The "hp" stanza above does not work with aCC (C++) and HP's ia64 # compilers, which have integrated preprocessors. The correct option # to use with these is +Maked; it writes dependencies to a file named # 'foo.d', which lands next to the object file, wherever that # happens to be. # Much of this is similar to the tru64 case; see comments there. set_dir_from "$object" set_base_from "$object" if test "$libtool" = yes; then tmpdepfile1=$dir$base.d tmpdepfile2=$dir.libs/$base.d "$@" -Wc,+Maked else tmpdepfile1=$dir$base.d tmpdepfile2=$dir$base.d "$@" +Maked fi stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile1" "$tmpdepfile2" exit $stat fi for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" do test -f "$tmpdepfile" && break done if test -f "$tmpdepfile"; then sed -e "s,^.*\.[$lower]*:,$object:," "$tmpdepfile" > "$depfile" # Add 'dependent.h:' lines. sed -ne '2,${ s/^ *// s/ \\*$// s/$/:/ p }' "$tmpdepfile" >> "$depfile" else make_dummy_depfile fi rm -f "$tmpdepfile" "$tmpdepfile2" ;; tru64) # The Tru64 compiler uses -MD to generate dependencies as a side # effect. 'cc -MD -o foo.o ...' puts the dependencies into 'foo.o.d'. # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put # dependencies in 'foo.d' instead, so we check for that too. # Subdirectories are respected. set_dir_from "$object" set_base_from "$object" if test "$libtool" = yes; then # Libtool generates 2 separate objects for the 2 libraries. These # two compilations output dependencies in $dir.libs/$base.o.d and # in $dir$base.o.d. We have to check for both files, because # one of the two compilations can be disabled. We should prefer # $dir$base.o.d over $dir.libs/$base.o.d because the latter is # automatically cleaned when .libs/ is deleted, while ignoring # the former would cause a distcleancheck panic. tmpdepfile1=$dir$base.o.d # libtool 1.5 tmpdepfile2=$dir.libs/$base.o.d # Likewise. tmpdepfile3=$dir.libs/$base.d # Compaq CCC V6.2-504 "$@" -Wc,-MD else tmpdepfile1=$dir$base.d tmpdepfile2=$dir$base.d tmpdepfile3=$dir$base.d "$@" -MD fi stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" exit $stat fi for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" do test -f "$tmpdepfile" && break done # Same post-processing that is required for AIX mode. aix_post_process_depfile ;; msvc7) if test "$libtool" = yes; then showIncludes=-Wc,-showIncludes else showIncludes=-showIncludes fi "$@" $showIncludes > "$tmpdepfile" stat=$? grep -v '^Note: including file: ' "$tmpdepfile" if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" echo "$object : \\" > "$depfile" # The first sed program below extracts the file names and escapes # backslashes for cygpath. The second sed program outputs the file # name when reading, but also accumulates all include files in the # hold buffer in order to output them again at the end. This only # works with sed implementations that can handle large buffers. sed < "$tmpdepfile" -n ' /^Note: including file: *\(.*\)/ { s//\1/ s/\\/\\\\/g p }' | $cygpath_u | sort -u | sed -n ' s/ /\\ /g s/\(.*\)/'"$tab"'\1 \\/p s/.\(.*\) \\/\1:/ H $ { s/.*/'"$tab"'/ G p }' >> "$depfile" echo >> "$depfile" # make sure the fragment doesn't end with a backslash rm -f "$tmpdepfile" ;; msvc7msys) # This case exists only to let depend.m4 do its work. It works by # looking at the text of this script. This case will never be run, # since it is checked for above. exit 1 ;; #nosideeffect) # This comment above is used by automake to tell side-effect # dependency tracking mechanisms from slower ones. dashmstdout) # Important note: in order to support this mode, a compiler *must* # always write the preprocessed file to stdout, regardless of -o. "$@" || exit $? # Remove the call to Libtool. if test "$libtool" = yes; then while test "X$1" != 'X--mode=compile'; do shift done shift fi # Remove '-o $object'. IFS=" " for arg do case $arg in -o) shift ;; $object) shift ;; *) set fnord "$@" "$arg" shift # fnord shift # $arg ;; esac done test -z "$dashmflag" && dashmflag=-M # Require at least two characters before searching for ':' # in the target name. This is to cope with DOS-style filenames: # a dependency such as 'c:/foo/bar' could be seen as target 'c' otherwise. "$@" $dashmflag | sed "s|^[$tab ]*[^:$tab ][^:][^:]*:[$tab ]*|$object: |" > "$tmpdepfile" rm -f "$depfile" cat < "$tmpdepfile" > "$depfile" # Some versions of the HPUX 10.20 sed can't process this sed invocation # correctly. Breaking it into two sed invocations is a workaround. tr ' ' "$nl" < "$tmpdepfile" \ | sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' \ | sed -e 's/$/ :/' >> "$depfile" rm -f "$tmpdepfile" ;; dashXmstdout) # This case only exists to satisfy depend.m4. It is never actually # run, as this mode is specially recognized in the preamble. exit 1 ;; makedepend) "$@" || exit $? # Remove any Libtool call if test "$libtool" = yes; then while test "X$1" != 'X--mode=compile'; do shift done shift fi # X makedepend shift cleared=no eat=no for arg do case $cleared in no) set ""; shift cleared=yes ;; esac if test $eat = yes; then eat=no continue fi case "$arg" in -D*|-I*) set fnord "$@" "$arg"; shift ;; # Strip any option that makedepend may not understand. Remove # the object too, otherwise makedepend will parse it as a source file. -arch) eat=yes ;; -*|$object) ;; *) set fnord "$@" "$arg"; shift ;; esac done obj_suffix=`echo "$object" | sed 's/^.*\././'` touch "$tmpdepfile" ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@" rm -f "$depfile" # makedepend may prepend the VPATH from the source file name to the object. # No need to regex-escape $object, excess matching of '.' is harmless. sed "s|^.*\($object *:\)|\1|" "$tmpdepfile" > "$depfile" # Some versions of the HPUX 10.20 sed can't process the last invocation # correctly. Breaking it into two sed invocations is a workaround. sed '1,2d' "$tmpdepfile" \ | tr ' ' "$nl" \ | sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' \ | sed -e 's/$/ :/' >> "$depfile" rm -f "$tmpdepfile" "$tmpdepfile".bak ;; cpp) # Important note: in order to support this mode, a compiler *must* # always write the preprocessed file to stdout. "$@" || exit $? # Remove the call to Libtool. if test "$libtool" = yes; then while test "X$1" != 'X--mode=compile'; do shift done shift fi # Remove '-o $object'. IFS=" " for arg do case $arg in -o) shift ;; $object) shift ;; *) set fnord "$@" "$arg" shift # fnord shift # $arg ;; esac done "$@" -E \ | sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \ -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \ | sed '$ s: \\$::' > "$tmpdepfile" rm -f "$depfile" echo "$object : \\" > "$depfile" cat < "$tmpdepfile" >> "$depfile" sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile" rm -f "$tmpdepfile" ;; msvisualcpp) # Important note: in order to support this mode, a compiler *must* # always write the preprocessed file to stdout. "$@" || exit $? # Remove the call to Libtool. if test "$libtool" = yes; then while test "X$1" != 'X--mode=compile'; do shift done shift fi IFS=" " for arg do case "$arg" in -o) shift ;; $object) shift ;; "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI") set fnord "$@" shift shift ;; *) set fnord "$@" "$arg" shift shift ;; esac done "$@" -E 2>/dev/null | sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile" rm -f "$depfile" echo "$object : \\" > "$depfile" sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::'"$tab"'\1 \\:p' >> "$depfile" echo "$tab" >> "$depfile" sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile" rm -f "$tmpdepfile" ;; msvcmsys) # This case exists only to let depend.m4 do its work. It works by # looking at the text of this script. This case will never be run, # since it is checked for above. exit 1 ;; none) exec "$@" ;; *) echo "Unknown depmode $depmode" 1>&2 exit 1 ;; esac exit 0 # Local Variables: # mode: shell-script # sh-indentation: 2 # eval: (add-hook 'write-file-hooks 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" # time-stamp-time-zone: "UTC0" # time-stamp-end: "; # UTC" # End: memcached-1.5.22/openbsd_priv.c0000664000175000017500000000163113575022500013270 00000000000000#include #include #include #include #include #include "memcached.h" /* * this section of code will drop all (OpenBSD) privileges including * those normally granted to all userland process (basic privileges). The * effect of this is that after running this code, the process will not able * to fork(), exec(), etc. See pledge(2) for more information. */ void drop_privileges() { extern char *__progname; if (settings.socketpath != NULL) { if (pledge("stdio unix", NULL) == -1) { fprintf(stderr, "%s: pledge: %s\n", __progname, strerror(errno)); exit(EXIT_FAILURE); } } else { if (pledge("stdio inet", NULL) == -1) { fprintf(stderr, "%s: pledge: %s\n", __progname, strerror(errno)); exit(EXIT_FAILURE); } } } void setup_privilege_violations_handler(void) { // not needed } memcached-1.5.22/stats_prefix.c0000664000175000017500000001044313611703161013312 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ /* Author: Steven Grimm */ #include "memcached.h" #include #include #include #include /* Hash table that uses the global hash function */ static PREFIX_STATS *prefix_stats[PREFIX_HASH_SIZE]; static char prefix_delimiter; static int num_prefixes = 0; static int total_prefix_size = 0; void stats_prefix_init(char delimiter) { prefix_delimiter = delimiter; memset(prefix_stats, 0, sizeof(prefix_stats)); } void stats_prefix_clear(void) { int i; for (i = 0; i < PREFIX_HASH_SIZE; i++) { PREFIX_STATS *cur, *next; for (cur = prefix_stats[i]; cur != NULL; cur = next) { next = cur->next; free(cur->prefix); free(cur); } prefix_stats[i] = NULL; } num_prefixes = 0; total_prefix_size = 0; } PREFIX_STATS *stats_prefix_find(const char *key, const size_t nkey) { PREFIX_STATS *pfs; uint32_t hashval; size_t length; bool bailout = true; assert(key != NULL); for (length = 0; length < nkey && key[length] != '\0'; length++) { if (key[length] == prefix_delimiter) { bailout = false; break; } } if (bailout) { return NULL; } hashval = hash(key, length) % PREFIX_HASH_SIZE; for (pfs = prefix_stats[hashval]; NULL != pfs; pfs = pfs->next) { if (strncmp(pfs->prefix, key, length) == 0) return pfs; } pfs = calloc(sizeof(PREFIX_STATS), 1); if (NULL == pfs) { perror("Can't allocate space for stats structure: calloc"); return NULL; } pfs->prefix = malloc(length + 1); if (NULL == pfs->prefix) { perror("Can't allocate space for copy of prefix: malloc"); free(pfs); return NULL; } strncpy(pfs->prefix, key, length); pfs->prefix[length] = '\0'; /* because strncpy() sucks */ pfs->prefix_len = length; pfs->next = prefix_stats[hashval]; prefix_stats[hashval] = pfs; num_prefixes++; total_prefix_size += length; return pfs; } void stats_prefix_record_get(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); pfs = stats_prefix_find(key, nkey); if (NULL != pfs) { pfs->num_gets++; if (is_hit) { pfs->num_hits++; } } STATS_UNLOCK(); } void stats_prefix_record_delete(const char *key, const size_t nkey) { PREFIX_STATS *pfs; STATS_LOCK(); pfs = stats_prefix_find(key, nkey); if (NULL != pfs) { pfs->num_deletes++; } STATS_UNLOCK(); } void stats_prefix_record_set(const char *key, const size_t nkey) { PREFIX_STATS *pfs; STATS_LOCK(); pfs = stats_prefix_find(key, nkey); if (NULL != pfs) { pfs->num_sets++; } STATS_UNLOCK(); } char *stats_prefix_dump(int *length) { const char *format = "PREFIX %s get %llu hit %llu set %llu del %llu\r\n"; PREFIX_STATS *pfs; char *buf; int i, pos; size_t size = 0, written = 0, total_written = 0; /* * Figure out how big the buffer needs to be. This is the sum of the * lengths of the prefixes themselves, plus the size of one copy of * the per-prefix output with 20-digit values for all the counts, * plus space for the "END" at the end. */ STATS_LOCK(); size = strlen(format) + total_prefix_size + num_prefixes * (strlen(format) - 2 /* %s */ + 4 * (20 - 4)) /* %llu replaced by 20-digit num */ + sizeof("END\r\n"); buf = malloc(size); if (NULL == buf) { perror("Can't allocate stats response: malloc"); STATS_UNLOCK(); return NULL; } pos = 0; for (i = 0; i < PREFIX_HASH_SIZE; i++) { for (pfs = prefix_stats[i]; NULL != pfs; pfs = pfs->next) { written = snprintf(buf + pos, size-pos, format, pfs->prefix, pfs->num_gets, pfs->num_hits, pfs->num_sets, pfs->num_deletes); pos += written; total_written += written; assert(total_written < size); } } STATS_UNLOCK(); memcpy(buf + pos, "END\r\n", 6); *length = pos + 5; return buf; } memcached-1.5.22/protocol_binary.h0000664000175000017500000004021513461170555014021 00000000000000/* * Copyright (c) <2008>, Sun Microsystems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY SUN MICROSYSTEMS, INC. ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL SUN MICROSYSTEMS, INC. BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * Summary: Constants used by to implement the binary protocol. * * Copy: See Copyright for the status of this software. * * Author: Trond Norbye */ #ifndef PROTOCOL_BINARY_H #define PROTOCOL_BINARY_H /** * This file contains definitions of the constants and packet formats * defined in the binary specification. Please note that you _MUST_ remember * to convert each multibyte field to / from network byte order to / from * host order. */ #ifdef __cplusplus extern "C" { #endif /** * Definition of the legal "magic" values used in a packet. * See section 3.1 Magic byte */ typedef enum { PROTOCOL_BINARY_REQ = 0x80, PROTOCOL_BINARY_RES = 0x81 } protocol_binary_magic; /** * Definition of the valid response status numbers. * See section 3.2 Response Status */ typedef enum { PROTOCOL_BINARY_RESPONSE_SUCCESS = 0x00, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT = 0x01, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS = 0x02, PROTOCOL_BINARY_RESPONSE_E2BIG = 0x03, PROTOCOL_BINARY_RESPONSE_EINVAL = 0x04, PROTOCOL_BINARY_RESPONSE_NOT_STORED = 0x05, PROTOCOL_BINARY_RESPONSE_DELTA_BADVAL = 0x06, PROTOCOL_BINARY_RESPONSE_AUTH_ERROR = 0x20, PROTOCOL_BINARY_RESPONSE_AUTH_CONTINUE = 0x21, PROTOCOL_BINARY_RESPONSE_UNKNOWN_COMMAND = 0x81, PROTOCOL_BINARY_RESPONSE_ENOMEM = 0x82 } protocol_binary_response_status; /** * Definition of the different command opcodes. * See section 3.3 Command Opcodes */ typedef enum { PROTOCOL_BINARY_CMD_GET = 0x00, PROTOCOL_BINARY_CMD_SET = 0x01, PROTOCOL_BINARY_CMD_ADD = 0x02, PROTOCOL_BINARY_CMD_REPLACE = 0x03, PROTOCOL_BINARY_CMD_DELETE = 0x04, PROTOCOL_BINARY_CMD_INCREMENT = 0x05, PROTOCOL_BINARY_CMD_DECREMENT = 0x06, PROTOCOL_BINARY_CMD_QUIT = 0x07, PROTOCOL_BINARY_CMD_FLUSH = 0x08, PROTOCOL_BINARY_CMD_GETQ = 0x09, PROTOCOL_BINARY_CMD_NOOP = 0x0a, PROTOCOL_BINARY_CMD_VERSION = 0x0b, PROTOCOL_BINARY_CMD_GETK = 0x0c, PROTOCOL_BINARY_CMD_GETKQ = 0x0d, PROTOCOL_BINARY_CMD_APPEND = 0x0e, PROTOCOL_BINARY_CMD_PREPEND = 0x0f, PROTOCOL_BINARY_CMD_STAT = 0x10, PROTOCOL_BINARY_CMD_SETQ = 0x11, PROTOCOL_BINARY_CMD_ADDQ = 0x12, PROTOCOL_BINARY_CMD_REPLACEQ = 0x13, PROTOCOL_BINARY_CMD_DELETEQ = 0x14, PROTOCOL_BINARY_CMD_INCREMENTQ = 0x15, PROTOCOL_BINARY_CMD_DECREMENTQ = 0x16, PROTOCOL_BINARY_CMD_QUITQ = 0x17, PROTOCOL_BINARY_CMD_FLUSHQ = 0x18, PROTOCOL_BINARY_CMD_APPENDQ = 0x19, PROTOCOL_BINARY_CMD_PREPENDQ = 0x1a, PROTOCOL_BINARY_CMD_TOUCH = 0x1c, PROTOCOL_BINARY_CMD_GAT = 0x1d, PROTOCOL_BINARY_CMD_GATQ = 0x1e, PROTOCOL_BINARY_CMD_GATK = 0x23, PROTOCOL_BINARY_CMD_GATKQ = 0x24, PROTOCOL_BINARY_CMD_SASL_LIST_MECHS = 0x20, PROTOCOL_BINARY_CMD_SASL_AUTH = 0x21, PROTOCOL_BINARY_CMD_SASL_STEP = 0x22, /* These commands are used for range operations and exist within * this header for use in other projects. Range operations are * not expected to be implemented in the memcached server itself. */ PROTOCOL_BINARY_CMD_RGET = 0x30, PROTOCOL_BINARY_CMD_RSET = 0x31, PROTOCOL_BINARY_CMD_RSETQ = 0x32, PROTOCOL_BINARY_CMD_RAPPEND = 0x33, PROTOCOL_BINARY_CMD_RAPPENDQ = 0x34, PROTOCOL_BINARY_CMD_RPREPEND = 0x35, PROTOCOL_BINARY_CMD_RPREPENDQ = 0x36, PROTOCOL_BINARY_CMD_RDELETE = 0x37, PROTOCOL_BINARY_CMD_RDELETEQ = 0x38, PROTOCOL_BINARY_CMD_RINCR = 0x39, PROTOCOL_BINARY_CMD_RINCRQ = 0x3a, PROTOCOL_BINARY_CMD_RDECR = 0x3b, PROTOCOL_BINARY_CMD_RDECRQ = 0x3c /* End Range operations */ } protocol_binary_command; /** * Definition of the data types in the packet * See section 3.4 Data Types */ typedef enum { PROTOCOL_BINARY_RAW_BYTES = 0x00 } protocol_binary_datatypes; /** * Definition of the header structure for a request packet. * See section 2 */ typedef union { struct { uint8_t magic; uint8_t opcode; uint16_t keylen; uint8_t extlen; uint8_t datatype; uint16_t reserved; uint32_t bodylen; uint32_t opaque; uint64_t cas; } request; uint8_t bytes[24]; } protocol_binary_request_header; /** * Definition of the header structure for a response packet. * See section 2 */ typedef union { struct { uint8_t magic; uint8_t opcode; uint16_t keylen; uint8_t extlen; uint8_t datatype; uint16_t status; uint32_t bodylen; uint32_t opaque; uint64_t cas; } response; uint8_t bytes[24]; } protocol_binary_response_header; /** * Definition of a request-packet containing no extras */ typedef union { struct { protocol_binary_request_header header; } message; uint8_t bytes[sizeof(protocol_binary_request_header)]; } protocol_binary_request_no_extras; /** * Definition of a response-packet containing no extras */ typedef union { struct { protocol_binary_response_header header; } message; uint8_t bytes[sizeof(protocol_binary_response_header)]; } protocol_binary_response_no_extras; /** * Definition of the packet used by the get, getq, getk and getkq command. * See section 4 */ typedef protocol_binary_request_no_extras protocol_binary_request_get; typedef protocol_binary_request_no_extras protocol_binary_request_getq; typedef protocol_binary_request_no_extras protocol_binary_request_getk; typedef protocol_binary_request_no_extras protocol_binary_request_getkq; /** * Definition of the packet returned from a successful get, getq, getk and * getkq. * See section 4 */ typedef union { struct { protocol_binary_response_header header; struct { uint32_t flags; } body; } message; uint8_t bytes[sizeof(protocol_binary_response_header) + 4]; } protocol_binary_response_get; typedef protocol_binary_response_get protocol_binary_response_getq; typedef protocol_binary_response_get protocol_binary_response_getk; typedef protocol_binary_response_get protocol_binary_response_getkq; /** * Definition of the packet used by the delete command * See section 4 */ typedef protocol_binary_request_no_extras protocol_binary_request_delete; /** * Definition of the packet returned by the delete command * See section 4 */ typedef protocol_binary_response_no_extras protocol_binary_response_delete; /** * Definition of the packet used by the flush command * See section 4 * Please note that the expiration field is optional, so remember to see * check the header.bodysize to see if it is present. */ typedef union { struct { protocol_binary_request_header header; struct { uint32_t expiration; } body; } message; uint8_t bytes[sizeof(protocol_binary_request_header) + 4]; } protocol_binary_request_flush; /** * Definition of the packet returned by the flush command * See section 4 */ typedef protocol_binary_response_no_extras protocol_binary_response_flush; /** * Definition of the packet used by set, add and replace * See section 4 */ typedef union { struct { protocol_binary_request_header header; struct { uint32_t flags; uint32_t expiration; } body; } message; uint8_t bytes[sizeof(protocol_binary_request_header) + 8]; } protocol_binary_request_set; typedef protocol_binary_request_set protocol_binary_request_add; typedef protocol_binary_request_set protocol_binary_request_replace; /** * Definition of the packet returned by set, add and replace * See section 4 */ typedef protocol_binary_response_no_extras protocol_binary_response_set; typedef protocol_binary_response_no_extras protocol_binary_response_add; typedef protocol_binary_response_no_extras protocol_binary_response_replace; /** * Definition of the noop packet * See section 4 */ typedef protocol_binary_request_no_extras protocol_binary_request_noop; /** * Definition of the packet returned by the noop command * See section 4 */ typedef protocol_binary_response_no_extras protocol_binary_response_noop; /** * Definition of the structure used by the increment and decrement * command. * See section 4 */ typedef union { struct { protocol_binary_request_header header; struct { uint64_t delta; uint64_t initial; uint32_t expiration; } body; } message; uint8_t bytes[sizeof(protocol_binary_request_header) + 20]; } protocol_binary_request_incr; typedef protocol_binary_request_incr protocol_binary_request_decr; /** * Definition of the response from an incr or decr command * command. * See section 4 */ typedef union { struct { protocol_binary_response_header header; struct { uint64_t value; } body; } message; uint8_t bytes[sizeof(protocol_binary_response_header) + 8]; } protocol_binary_response_incr; typedef protocol_binary_response_incr protocol_binary_response_decr; /** * Definition of the quit * See section 4 */ typedef protocol_binary_request_no_extras protocol_binary_request_quit; /** * Definition of the packet returned by the quit command * See section 4 */ typedef protocol_binary_response_no_extras protocol_binary_response_quit; /** * Definition of the packet used by append and prepend command * See section 4 */ typedef protocol_binary_request_no_extras protocol_binary_request_append; typedef protocol_binary_request_no_extras protocol_binary_request_prepend; /** * Definition of the packet returned from a successful append or prepend * See section 4 */ typedef protocol_binary_response_no_extras protocol_binary_response_append; typedef protocol_binary_response_no_extras protocol_binary_response_prepend; /** * Definition of the packet used by the version command * See section 4 */ typedef protocol_binary_request_no_extras protocol_binary_request_version; /** * Definition of the packet returned from a successful version command * See section 4 */ typedef protocol_binary_response_no_extras protocol_binary_response_version; /** * Definition of the packet used by the stats command. * See section 4 */ typedef protocol_binary_request_no_extras protocol_binary_request_stats; /** * Definition of the packet returned from a successful stats command * See section 4 */ typedef protocol_binary_response_no_extras protocol_binary_response_stats; /** * Definition of the packet used by the touch command. */ typedef union { struct { protocol_binary_request_header header; struct { uint32_t expiration; } body; } message; uint8_t bytes[sizeof(protocol_binary_request_header) + 4]; } protocol_binary_request_touch; /** * Definition of the packet returned from the touch command */ typedef protocol_binary_response_no_extras protocol_binary_response_touch; /** * Definition of the packet used by the GAT(Q) command. */ typedef union { struct { protocol_binary_request_header header; struct { uint32_t expiration; } body; } message; uint8_t bytes[sizeof(protocol_binary_request_header) + 4]; } protocol_binary_request_gat; typedef protocol_binary_request_gat protocol_binary_request_gatq; typedef protocol_binary_request_gat protocol_binary_request_gatk; typedef protocol_binary_request_gat protocol_binary_request_gatkq; /** * Definition of the packet returned from the GAT(Q) */ typedef protocol_binary_response_get protocol_binary_response_gat; typedef protocol_binary_response_get protocol_binary_response_gatq; typedef protocol_binary_response_get protocol_binary_response_gatk; typedef protocol_binary_response_get protocol_binary_response_gatkq; /** * Definition of a request for a range operation. * See http://code.google.com/p/memcached/wiki/RangeOps * * These types are used for range operations and exist within * this header for use in other projects. Range operations are * not expected to be implemented in the memcached server itself. */ typedef union { struct { protocol_binary_response_header header; struct { uint16_t size; uint8_t reserved; uint8_t flags; uint32_t max_results; } body; } message; uint8_t bytes[sizeof(protocol_binary_request_header) + 4]; } protocol_binary_request_rangeop; typedef protocol_binary_request_rangeop protocol_binary_request_rget; typedef protocol_binary_request_rangeop protocol_binary_request_rset; typedef protocol_binary_request_rangeop protocol_binary_request_rsetq; typedef protocol_binary_request_rangeop protocol_binary_request_rappend; typedef protocol_binary_request_rangeop protocol_binary_request_rappendq; typedef protocol_binary_request_rangeop protocol_binary_request_rprepend; typedef protocol_binary_request_rangeop protocol_binary_request_rprependq; typedef protocol_binary_request_rangeop protocol_binary_request_rdelete; typedef protocol_binary_request_rangeop protocol_binary_request_rdeleteq; typedef protocol_binary_request_rangeop protocol_binary_request_rincr; typedef protocol_binary_request_rangeop protocol_binary_request_rincrq; typedef protocol_binary_request_rangeop protocol_binary_request_rdecr; typedef protocol_binary_request_rangeop protocol_binary_request_rdecrq; #ifdef __cplusplus } #endif #endif /* PROTOCOL_BINARY_H */ memcached-1.5.22/itoa_ljust.h0000664000175000017500000000146613461170555012776 00000000000000#ifndef ITOA_LJUST_H #define ITOA_LJUST_H //=== itoa_ljust.h - Fast integer to ascii conversion // // Fast and simple integer to ASCII conversion: // // - 32 and 64-bit integers // - signed and unsigned // - user supplied buffer must be large enough for all decimal digits // in value plus minus sign if negative // - left-justified // - NUL terminated // - return value is pointer to NUL terminator // // Copyright (c) 2016 Arturo Martin-de-Nicolas // arturomdn@gmail.com // https://github.com/amdn/itoa_ljust/ //===----------------------------------------------------------------------===// #include char* itoa_u32(uint32_t u, char* buffer); char* itoa_32( int32_t i, char* buffer); char* itoa_u64(uint64_t u, char* buffer); char* itoa_64( int64_t i, char* buffer); #endif // ITOA_LJUST_H memcached-1.5.22/items.c0000664000175000017500000017452013615372510011733 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #include "memcached.h" #include "bipbuffer.h" #include "slab_automove.h" #ifdef EXTSTORE #include "storage.h" #include "slab_automove_extstore.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Forward Declarations */ static void item_link_q(item *it); static void item_unlink_q(item *it); static unsigned int lru_type_map[4] = {HOT_LRU, WARM_LRU, COLD_LRU, TEMP_LRU}; #define LARGEST_ID POWER_LARGEST typedef struct { uint64_t evicted; uint64_t evicted_nonzero; uint64_t reclaimed; uint64_t outofmemory; uint64_t tailrepairs; uint64_t expired_unfetched; /* items reclaimed but never touched */ uint64_t evicted_unfetched; /* items evicted but never touched */ uint64_t evicted_active; /* items evicted that should have been shuffled */ uint64_t crawler_reclaimed; uint64_t crawler_items_checked; uint64_t lrutail_reflocked; uint64_t moves_to_cold; uint64_t moves_to_warm; uint64_t moves_within_lru; uint64_t direct_reclaims; uint64_t hits_to_hot; uint64_t hits_to_warm; uint64_t hits_to_cold; uint64_t hits_to_temp; uint64_t mem_requested; rel_time_t evicted_time; } itemstats_t; static item *heads[LARGEST_ID]; static item *tails[LARGEST_ID]; static itemstats_t itemstats[LARGEST_ID]; static unsigned int sizes[LARGEST_ID]; static uint64_t sizes_bytes[LARGEST_ID]; static unsigned int *stats_sizes_hist = NULL; static uint64_t stats_sizes_cas_min = 0; static int stats_sizes_buckets = 0; static uint64_t cas_id = 0; static volatile int do_run_lru_maintainer_thread = 0; static int lru_maintainer_initialized = 0; static pthread_mutex_t lru_maintainer_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t cas_id_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t stats_sizes_lock = PTHREAD_MUTEX_INITIALIZER; void item_stats_reset(void) { int i; for (i = 0; i < LARGEST_ID; i++) { pthread_mutex_lock(&lru_locks[i]); memset(&itemstats[i], 0, sizeof(itemstats_t)); pthread_mutex_unlock(&lru_locks[i]); } } /* called with class lru lock held */ void do_item_stats_add_crawl(const int i, const uint64_t reclaimed, const uint64_t unfetched, const uint64_t checked) { itemstats[i].crawler_reclaimed += reclaimed; itemstats[i].expired_unfetched += unfetched; itemstats[i].crawler_items_checked += checked; } typedef struct _lru_bump_buf { struct _lru_bump_buf *prev; struct _lru_bump_buf *next; pthread_mutex_t mutex; bipbuf_t *buf; uint64_t dropped; } lru_bump_buf; typedef struct { item *it; uint32_t hv; } lru_bump_entry; static lru_bump_buf *bump_buf_head = NULL; static lru_bump_buf *bump_buf_tail = NULL; static pthread_mutex_t bump_buf_lock = PTHREAD_MUTEX_INITIALIZER; /* TODO: tunable? Need bench results */ #define LRU_BUMP_BUF_SIZE 8192 static bool lru_bump_async(lru_bump_buf *b, item *it, uint32_t hv); static uint64_t lru_total_bumps_dropped(void); /* Get the next CAS id for a new item. */ /* TODO: refactor some atomics for this. */ uint64_t get_cas_id(void) { pthread_mutex_lock(&cas_id_lock); uint64_t next_id = ++cas_id; pthread_mutex_unlock(&cas_id_lock); return next_id; } void set_cas_id(uint64_t new_cas) { pthread_mutex_lock(&cas_id_lock); cas_id = new_cas; pthread_mutex_unlock(&cas_id_lock); } int item_is_flushed(item *it) { rel_time_t oldest_live = settings.oldest_live; uint64_t cas = ITEM_get_cas(it); uint64_t oldest_cas = settings.oldest_cas; if (oldest_live == 0 || oldest_live > current_time) return 0; if ((it->time <= oldest_live) || (oldest_cas != 0 && cas != 0 && cas < oldest_cas)) { return 1; } return 0; } /* must be locked before call */ unsigned int do_get_lru_size(uint32_t id) { return sizes[id]; } /* Enable this for reference-count debugging. */ #if 0 # define DEBUG_REFCNT(it,op) \ fprintf(stderr, "item %x refcnt(%c) %d %c%c%c\n", \ it, op, it->refcount, \ (it->it_flags & ITEM_LINKED) ? 'L' : ' ', \ (it->it_flags & ITEM_SLABBED) ? 'S' : ' ') #else # define DEBUG_REFCNT(it,op) while(0) #endif /** * Generates the variable-sized part of the header for an object. * * nkey - The length of the key * flags - key flags * nbytes - Number of bytes to hold value and addition CRLF terminator * suffix - Buffer for the "VALUE" line suffix (flags, size). * nsuffix - The length of the suffix is stored here. * * Returns the total size of the header. */ static size_t item_make_header(const uint8_t nkey, const unsigned int flags, const int nbytes, char *suffix, uint8_t *nsuffix) { if (flags == 0) { *nsuffix = 0; } else { *nsuffix = sizeof(flags); } return sizeof(item) + nkey + *nsuffix + nbytes; } item *do_item_alloc_pull(const size_t ntotal, const unsigned int id) { item *it = NULL; int i; /* If no memory is available, attempt a direct LRU juggle/eviction */ /* This is a race in order to simplify lru_pull_tail; in cases where * locked items are on the tail, you want them to fall out and cause * occasional OOM's, rather than internally work around them. * This also gives one fewer code path for slab alloc/free */ for (i = 0; i < 10; i++) { /* Try to reclaim memory first */ if (!settings.lru_segmented) { lru_pull_tail(id, COLD_LRU, 0, 0, 0, NULL); } it = slabs_alloc(ntotal, id, 0); if (it == NULL) { // We send '0' in for "total_bytes" as this routine is always // pulling to evict, or forcing HOT -> COLD migration. // As of this writing, total_bytes isn't at all used with COLD_LRU. if (lru_pull_tail(id, COLD_LRU, 0, LRU_PULL_EVICT, 0, NULL) <= 0) { if (settings.lru_segmented) { lru_pull_tail(id, HOT_LRU, 0, 0, 0, NULL); } else { break; } } } else { break; } } if (i > 0) { pthread_mutex_lock(&lru_locks[id]); itemstats[id].direct_reclaims += i; pthread_mutex_unlock(&lru_locks[id]); } return it; } /* Chain another chunk onto this chunk. */ /* slab mover: if it finds a chunk without ITEM_CHUNK flag, and no ITEM_LINKED * flag, it counts as busy and skips. * I think it might still not be safe to do linking outside of the slab lock */ item_chunk *do_item_alloc_chunk(item_chunk *ch, const size_t bytes_remain) { // TODO: Should be a cleaner way of finding real size with slabber calls size_t size = bytes_remain + sizeof(item_chunk); if (size > settings.slab_chunk_size_max) size = settings.slab_chunk_size_max; unsigned int id = slabs_clsid(size); item_chunk *nch = (item_chunk *) do_item_alloc_pull(size, id); if (nch == NULL) return NULL; // link in. // ITEM_CHUNK[ED] bits need to be protected by the slabs lock. slabs_mlock(); nch->head = ch->head; ch->next = nch; nch->prev = ch; nch->next = 0; nch->used = 0; nch->slabs_clsid = id; nch->size = size - sizeof(item_chunk); nch->it_flags |= ITEM_CHUNK; slabs_munlock(); return nch; } item *do_item_alloc(char *key, const size_t nkey, const unsigned int flags, const rel_time_t exptime, const int nbytes) { uint8_t nsuffix; item *it = NULL; char suffix[40]; // Avoid potential underflows. if (nbytes < 2) return 0; size_t ntotal = item_make_header(nkey + 1, flags, nbytes, suffix, &nsuffix); if (settings.use_cas) { ntotal += sizeof(uint64_t); } unsigned int id = slabs_clsid(ntotal); unsigned int hdr_id = 0; if (id == 0) return 0; /* This is a large item. Allocate a header object now, lazily allocate * chunks while reading the upload. */ if (ntotal > settings.slab_chunk_size_max) { /* We still link this item into the LRU for the larger slab class, but * we're pulling a header from an entirely different slab class. The * free routines handle large items specifically. */ int htotal = nkey + 1 + nsuffix + sizeof(item) + sizeof(item_chunk); if (settings.use_cas) { htotal += sizeof(uint64_t); } #ifdef NEED_ALIGN // header chunk needs to be padded on some systems int remain = htotal % 8; if (remain != 0) { htotal += 8 - remain; } #endif hdr_id = slabs_clsid(htotal); it = do_item_alloc_pull(htotal, hdr_id); /* setting ITEM_CHUNKED is fine here because we aren't LINKED yet. */ if (it != NULL) it->it_flags |= ITEM_CHUNKED; } else { it = do_item_alloc_pull(ntotal, id); } if (it == NULL) { pthread_mutex_lock(&lru_locks[id]); itemstats[id].outofmemory++; pthread_mutex_unlock(&lru_locks[id]); return NULL; } assert(it->it_flags == 0 || it->it_flags == ITEM_CHUNKED); //assert(it != heads[id]); /* Refcount is seeded to 1 by slabs_alloc() */ it->next = it->prev = 0; /* Items are initially loaded into the HOT_LRU. This is '0' but I want at * least a note here. Compiler (hopefully?) optimizes this out. */ if (settings.temp_lru && exptime - current_time <= settings.temporary_ttl) { id |= TEMP_LRU; } else if (settings.lru_segmented) { id |= HOT_LRU; } else { /* There is only COLD in compat-mode */ id |= COLD_LRU; } it->slabs_clsid = id; DEBUG_REFCNT(it, '*'); it->it_flags |= settings.use_cas ? ITEM_CAS : 0; it->it_flags |= nsuffix != 0 ? ITEM_CFLAGS : 0; it->nkey = nkey; it->nbytes = nbytes; memcpy(ITEM_key(it), key, nkey); it->exptime = exptime; if (nsuffix > 0) { memcpy(ITEM_suffix(it), &flags, sizeof(flags)); } /* Initialize internal chunk. */ if (it->it_flags & ITEM_CHUNKED) { item_chunk *chunk = (item_chunk *) ITEM_schunk(it); chunk->next = 0; chunk->prev = 0; chunk->used = 0; chunk->size = 0; chunk->head = it; chunk->orig_clsid = hdr_id; } it->h_next = 0; return it; } void item_free(item *it) { size_t ntotal = ITEM_ntotal(it); unsigned int clsid; assert((it->it_flags & ITEM_LINKED) == 0); assert(it != heads[it->slabs_clsid]); assert(it != tails[it->slabs_clsid]); assert(it->refcount == 0); /* so slab size changer can tell later if item is already free or not */ clsid = ITEM_clsid(it); DEBUG_REFCNT(it, 'F'); slabs_free(it, ntotal, clsid); } /** * Returns true if an item will fit in the cache (its size does not exceed * the maximum for a cache entry.) */ bool item_size_ok(const size_t nkey, const int flags, const int nbytes) { char prefix[40]; uint8_t nsuffix; if (nbytes < 2) return false; size_t ntotal = item_make_header(nkey + 1, flags, nbytes, prefix, &nsuffix); if (settings.use_cas) { ntotal += sizeof(uint64_t); } return slabs_clsid(ntotal) != 0; } /* fixing stats/references during warm start */ void do_item_link_fixup(item *it) { item **head, **tail; int ntotal = ITEM_ntotal(it); uint32_t hv = hash(ITEM_key(it), it->nkey); assoc_insert(it, hv); head = &heads[it->slabs_clsid]; tail = &tails[it->slabs_clsid]; if (it->prev == 0 && *head == 0) *head = it; if (it->next == 0 && *tail == 0) *tail = it; sizes[it->slabs_clsid]++; sizes_bytes[it->slabs_clsid] += ntotal; STATS_LOCK(); stats_state.curr_bytes += ntotal; stats_state.curr_items += 1; stats.total_items += 1; STATS_UNLOCK(); item_stats_sizes_add(it); return; } static void do_item_link_q(item *it) { /* item is the new head */ item **head, **tail; assert((it->it_flags & ITEM_SLABBED) == 0); head = &heads[it->slabs_clsid]; tail = &tails[it->slabs_clsid]; assert(it != *head); assert((*head && *tail) || (*head == 0 && *tail == 0)); it->prev = 0; it->next = *head; if (it->next) it->next->prev = it; *head = it; if (*tail == 0) *tail = it; sizes[it->slabs_clsid]++; #ifdef EXTSTORE if (it->it_flags & ITEM_HDR) { sizes_bytes[it->slabs_clsid] += (ITEM_ntotal(it) - it->nbytes) + sizeof(item_hdr); } else { sizes_bytes[it->slabs_clsid] += ITEM_ntotal(it); } #else sizes_bytes[it->slabs_clsid] += ITEM_ntotal(it); #endif return; } static void item_link_q(item *it) { pthread_mutex_lock(&lru_locks[it->slabs_clsid]); do_item_link_q(it); pthread_mutex_unlock(&lru_locks[it->slabs_clsid]); } static void item_link_q_warm(item *it) { pthread_mutex_lock(&lru_locks[it->slabs_clsid]); do_item_link_q(it); itemstats[it->slabs_clsid].moves_to_warm++; pthread_mutex_unlock(&lru_locks[it->slabs_clsid]); } static void do_item_unlink_q(item *it) { item **head, **tail; head = &heads[it->slabs_clsid]; tail = &tails[it->slabs_clsid]; if (*head == it) { assert(it->prev == 0); *head = it->next; } if (*tail == it) { assert(it->next == 0); *tail = it->prev; } assert(it->next != it); assert(it->prev != it); if (it->next) it->next->prev = it->prev; if (it->prev) it->prev->next = it->next; sizes[it->slabs_clsid]--; #ifdef EXTSTORE if (it->it_flags & ITEM_HDR) { sizes_bytes[it->slabs_clsid] -= (ITEM_ntotal(it) - it->nbytes) + sizeof(item_hdr); } else { sizes_bytes[it->slabs_clsid] -= ITEM_ntotal(it); } #else sizes_bytes[it->slabs_clsid] -= ITEM_ntotal(it); #endif return; } static void item_unlink_q(item *it) { pthread_mutex_lock(&lru_locks[it->slabs_clsid]); do_item_unlink_q(it); pthread_mutex_unlock(&lru_locks[it->slabs_clsid]); } int do_item_link(item *it, const uint32_t hv) { MEMCACHED_ITEM_LINK(ITEM_key(it), it->nkey, it->nbytes); assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0); it->it_flags |= ITEM_LINKED; it->time = current_time; STATS_LOCK(); stats_state.curr_bytes += ITEM_ntotal(it); stats_state.curr_items += 1; stats.total_items += 1; STATS_UNLOCK(); /* Allocate a new CAS ID on link. */ ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0); assoc_insert(it, hv); item_link_q(it); refcount_incr(it); item_stats_sizes_add(it); return 1; } void do_item_unlink(item *it, const uint32_t hv) { MEMCACHED_ITEM_UNLINK(ITEM_key(it), it->nkey, it->nbytes); if ((it->it_flags & ITEM_LINKED) != 0) { it->it_flags &= ~ITEM_LINKED; STATS_LOCK(); stats_state.curr_bytes -= ITEM_ntotal(it); stats_state.curr_items -= 1; STATS_UNLOCK(); item_stats_sizes_remove(it); assoc_delete(ITEM_key(it), it->nkey, hv); item_unlink_q(it); do_item_remove(it); } } /* FIXME: Is it necessary to keep this copy/pasted code? */ void do_item_unlink_nolock(item *it, const uint32_t hv) { MEMCACHED_ITEM_UNLINK(ITEM_key(it), it->nkey, it->nbytes); if ((it->it_flags & ITEM_LINKED) != 0) { it->it_flags &= ~ITEM_LINKED; STATS_LOCK(); stats_state.curr_bytes -= ITEM_ntotal(it); stats_state.curr_items -= 1; STATS_UNLOCK(); item_stats_sizes_remove(it); assoc_delete(ITEM_key(it), it->nkey, hv); do_item_unlink_q(it); do_item_remove(it); } } void do_item_remove(item *it) { MEMCACHED_ITEM_REMOVE(ITEM_key(it), it->nkey, it->nbytes); assert((it->it_flags & ITEM_SLABBED) == 0); assert(it->refcount > 0); if (refcount_decr(it) == 0) { item_free(it); } } /* Bump the last accessed time, or relink if we're in compat mode */ void do_item_update(item *it) { MEMCACHED_ITEM_UPDATE(ITEM_key(it), it->nkey, it->nbytes); /* Hits to COLD_LRU immediately move to WARM. */ if (settings.lru_segmented) { assert((it->it_flags & ITEM_SLABBED) == 0); if ((it->it_flags & ITEM_LINKED) != 0) { if (ITEM_lruid(it) == COLD_LRU && (it->it_flags & ITEM_ACTIVE)) { it->time = current_time; item_unlink_q(it); it->slabs_clsid = ITEM_clsid(it); it->slabs_clsid |= WARM_LRU; it->it_flags &= ~ITEM_ACTIVE; item_link_q_warm(it); } else { it->time = current_time; } } } else if (it->time < current_time - ITEM_UPDATE_INTERVAL) { assert((it->it_flags & ITEM_SLABBED) == 0); if ((it->it_flags & ITEM_LINKED) != 0) { it->time = current_time; item_unlink_q(it); item_link_q(it); } } } int do_item_replace(item *it, item *new_it, const uint32_t hv) { MEMCACHED_ITEM_REPLACE(ITEM_key(it), it->nkey, it->nbytes, ITEM_key(new_it), new_it->nkey, new_it->nbytes); assert((it->it_flags & ITEM_SLABBED) == 0); do_item_unlink(it, hv); return do_item_link(new_it, hv); } /*@null@*/ /* This is walking the line of violating lock order, but I think it's safe. * If the LRU lock is held, an item in the LRU cannot be wiped and freed. * The data could possibly be overwritten, but this is only accessing the * headers. * It may not be the best idea to leave it like this, but for now it's safe. */ char *item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes) { unsigned int memlimit = 2 * 1024 * 1024; /* 2MB max response size */ char *buffer; unsigned int bufcurr; item *it; unsigned int len; unsigned int shown = 0; char key_temp[KEY_MAX_LENGTH + 1]; char temp[512]; unsigned int id = slabs_clsid; id |= COLD_LRU; pthread_mutex_lock(&lru_locks[id]); it = heads[id]; buffer = malloc((size_t)memlimit); if (buffer == 0) { pthread_mutex_unlock(&lru_locks[id]); return NULL; } bufcurr = 0; while (it != NULL && (limit == 0 || shown < limit)) { assert(it->nkey <= KEY_MAX_LENGTH); if (it->nbytes == 0 && it->nkey == 0) { it = it->next; continue; } /* Copy the key since it may not be null-terminated in the struct */ strncpy(key_temp, ITEM_key(it), it->nkey); key_temp[it->nkey] = 0x00; /* terminate */ len = snprintf(temp, sizeof(temp), "ITEM %s [%d b; %llu s]\r\n", key_temp, it->nbytes - 2, it->exptime == 0 ? 0 : (unsigned long long)it->exptime + process_started); if (bufcurr + len + 6 > memlimit) /* 6 is END\r\n\0 */ break; memcpy(buffer + bufcurr, temp, len); bufcurr += len; shown++; it = it->next; } memcpy(buffer + bufcurr, "END\r\n", 6); bufcurr += 5; *bytes = bufcurr; pthread_mutex_unlock(&lru_locks[id]); return buffer; } /* With refactoring of the various stats code the automover won't need a * custom function here. */ void fill_item_stats_automove(item_stats_automove *am) { int n; for (n = 0; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) { item_stats_automove *cur = &am[n]; // outofmemory records into HOT int i = n | HOT_LRU; pthread_mutex_lock(&lru_locks[i]); cur->outofmemory = itemstats[i].outofmemory; pthread_mutex_unlock(&lru_locks[i]); // evictions and tail age are from COLD i = n | COLD_LRU; pthread_mutex_lock(&lru_locks[i]); cur->evicted = itemstats[i].evicted; if (tails[i]) { cur->age = current_time - tails[i]->time; } else { cur->age = 0; } pthread_mutex_unlock(&lru_locks[i]); } } void item_stats_totals(ADD_STAT add_stats, void *c) { itemstats_t totals; memset(&totals, 0, sizeof(itemstats_t)); int n; for (n = 0; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) { int x; int i; for (x = 0; x < 4; x++) { i = n | lru_type_map[x]; pthread_mutex_lock(&lru_locks[i]); totals.expired_unfetched += itemstats[i].expired_unfetched; totals.evicted_unfetched += itemstats[i].evicted_unfetched; totals.evicted_active += itemstats[i].evicted_active; totals.evicted += itemstats[i].evicted; totals.reclaimed += itemstats[i].reclaimed; totals.crawler_reclaimed += itemstats[i].crawler_reclaimed; totals.crawler_items_checked += itemstats[i].crawler_items_checked; totals.lrutail_reflocked += itemstats[i].lrutail_reflocked; totals.moves_to_cold += itemstats[i].moves_to_cold; totals.moves_to_warm += itemstats[i].moves_to_warm; totals.moves_within_lru += itemstats[i].moves_within_lru; totals.direct_reclaims += itemstats[i].direct_reclaims; pthread_mutex_unlock(&lru_locks[i]); } } APPEND_STAT("expired_unfetched", "%llu", (unsigned long long)totals.expired_unfetched); APPEND_STAT("evicted_unfetched", "%llu", (unsigned long long)totals.evicted_unfetched); if (settings.lru_maintainer_thread) { APPEND_STAT("evicted_active", "%llu", (unsigned long long)totals.evicted_active); } APPEND_STAT("evictions", "%llu", (unsigned long long)totals.evicted); APPEND_STAT("reclaimed", "%llu", (unsigned long long)totals.reclaimed); APPEND_STAT("crawler_reclaimed", "%llu", (unsigned long long)totals.crawler_reclaimed); APPEND_STAT("crawler_items_checked", "%llu", (unsigned long long)totals.crawler_items_checked); APPEND_STAT("lrutail_reflocked", "%llu", (unsigned long long)totals.lrutail_reflocked); if (settings.lru_maintainer_thread) { APPEND_STAT("moves_to_cold", "%llu", (unsigned long long)totals.moves_to_cold); APPEND_STAT("moves_to_warm", "%llu", (unsigned long long)totals.moves_to_warm); APPEND_STAT("moves_within_lru", "%llu", (unsigned long long)totals.moves_within_lru); APPEND_STAT("direct_reclaims", "%llu", (unsigned long long)totals.direct_reclaims); APPEND_STAT("lru_bumps_dropped", "%llu", (unsigned long long)lru_total_bumps_dropped()); } } void item_stats(ADD_STAT add_stats, void *c) { struct thread_stats thread_stats; threadlocal_stats_aggregate(&thread_stats); itemstats_t totals; int n; for (n = 0; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) { memset(&totals, 0, sizeof(itemstats_t)); int x; int i; unsigned int size = 0; unsigned int age = 0; unsigned int age_hot = 0; unsigned int age_warm = 0; unsigned int lru_size_map[4]; const char *fmt = "items:%d:%s"; char key_str[STAT_KEY_LEN]; char val_str[STAT_VAL_LEN]; int klen = 0, vlen = 0; for (x = 0; x < 4; x++) { i = n | lru_type_map[x]; pthread_mutex_lock(&lru_locks[i]); totals.evicted += itemstats[i].evicted; totals.evicted_nonzero += itemstats[i].evicted_nonzero; totals.outofmemory += itemstats[i].outofmemory; totals.tailrepairs += itemstats[i].tailrepairs; totals.reclaimed += itemstats[i].reclaimed; totals.expired_unfetched += itemstats[i].expired_unfetched; totals.evicted_unfetched += itemstats[i].evicted_unfetched; totals.evicted_active += itemstats[i].evicted_active; totals.crawler_reclaimed += itemstats[i].crawler_reclaimed; totals.crawler_items_checked += itemstats[i].crawler_items_checked; totals.lrutail_reflocked += itemstats[i].lrutail_reflocked; totals.moves_to_cold += itemstats[i].moves_to_cold; totals.moves_to_warm += itemstats[i].moves_to_warm; totals.moves_within_lru += itemstats[i].moves_within_lru; totals.direct_reclaims += itemstats[i].direct_reclaims; totals.mem_requested += sizes_bytes[i]; size += sizes[i]; lru_size_map[x] = sizes[i]; if (lru_type_map[x] == COLD_LRU && tails[i] != NULL) { age = current_time - tails[i]->time; } else if (lru_type_map[x] == HOT_LRU && tails[i] != NULL) { age_hot = current_time - tails[i]->time; } else if (lru_type_map[x] == WARM_LRU && tails[i] != NULL) { age_warm = current_time - tails[i]->time; } if (lru_type_map[x] == COLD_LRU) totals.evicted_time = itemstats[i].evicted_time; switch (lru_type_map[x]) { case HOT_LRU: totals.hits_to_hot = thread_stats.lru_hits[i]; break; case WARM_LRU: totals.hits_to_warm = thread_stats.lru_hits[i]; break; case COLD_LRU: totals.hits_to_cold = thread_stats.lru_hits[i]; break; case TEMP_LRU: totals.hits_to_temp = thread_stats.lru_hits[i]; break; } pthread_mutex_unlock(&lru_locks[i]); } if (size == 0) continue; APPEND_NUM_FMT_STAT(fmt, n, "number", "%u", size); if (settings.lru_maintainer_thread) { APPEND_NUM_FMT_STAT(fmt, n, "number_hot", "%u", lru_size_map[0]); APPEND_NUM_FMT_STAT(fmt, n, "number_warm", "%u", lru_size_map[1]); APPEND_NUM_FMT_STAT(fmt, n, "number_cold", "%u", lru_size_map[2]); if (settings.temp_lru) { APPEND_NUM_FMT_STAT(fmt, n, "number_temp", "%u", lru_size_map[3]); } APPEND_NUM_FMT_STAT(fmt, n, "age_hot", "%u", age_hot); APPEND_NUM_FMT_STAT(fmt, n, "age_warm", "%u", age_warm); } APPEND_NUM_FMT_STAT(fmt, n, "age", "%u", age); APPEND_NUM_FMT_STAT(fmt, n, "mem_requested", "%llu", (unsigned long long)totals.mem_requested); APPEND_NUM_FMT_STAT(fmt, n, "evicted", "%llu", (unsigned long long)totals.evicted); APPEND_NUM_FMT_STAT(fmt, n, "evicted_nonzero", "%llu", (unsigned long long)totals.evicted_nonzero); APPEND_NUM_FMT_STAT(fmt, n, "evicted_time", "%u", totals.evicted_time); APPEND_NUM_FMT_STAT(fmt, n, "outofmemory", "%llu", (unsigned long long)totals.outofmemory); APPEND_NUM_FMT_STAT(fmt, n, "tailrepairs", "%llu", (unsigned long long)totals.tailrepairs); APPEND_NUM_FMT_STAT(fmt, n, "reclaimed", "%llu", (unsigned long long)totals.reclaimed); APPEND_NUM_FMT_STAT(fmt, n, "expired_unfetched", "%llu", (unsigned long long)totals.expired_unfetched); APPEND_NUM_FMT_STAT(fmt, n, "evicted_unfetched", "%llu", (unsigned long long)totals.evicted_unfetched); if (settings.lru_maintainer_thread) { APPEND_NUM_FMT_STAT(fmt, n, "evicted_active", "%llu", (unsigned long long)totals.evicted_active); } APPEND_NUM_FMT_STAT(fmt, n, "crawler_reclaimed", "%llu", (unsigned long long)totals.crawler_reclaimed); APPEND_NUM_FMT_STAT(fmt, n, "crawler_items_checked", "%llu", (unsigned long long)totals.crawler_items_checked); APPEND_NUM_FMT_STAT(fmt, n, "lrutail_reflocked", "%llu", (unsigned long long)totals.lrutail_reflocked); if (settings.lru_maintainer_thread) { APPEND_NUM_FMT_STAT(fmt, n, "moves_to_cold", "%llu", (unsigned long long)totals.moves_to_cold); APPEND_NUM_FMT_STAT(fmt, n, "moves_to_warm", "%llu", (unsigned long long)totals.moves_to_warm); APPEND_NUM_FMT_STAT(fmt, n, "moves_within_lru", "%llu", (unsigned long long)totals.moves_within_lru); APPEND_NUM_FMT_STAT(fmt, n, "direct_reclaims", "%llu", (unsigned long long)totals.direct_reclaims); APPEND_NUM_FMT_STAT(fmt, n, "hits_to_hot", "%llu", (unsigned long long)totals.hits_to_hot); APPEND_NUM_FMT_STAT(fmt, n, "hits_to_warm", "%llu", (unsigned long long)totals.hits_to_warm); APPEND_NUM_FMT_STAT(fmt, n, "hits_to_cold", "%llu", (unsigned long long)totals.hits_to_cold); APPEND_NUM_FMT_STAT(fmt, n, "hits_to_temp", "%llu", (unsigned long long)totals.hits_to_temp); } } /* getting here means both ascii and binary terminators fit */ add_stats(NULL, 0, NULL, 0, c); } bool item_stats_sizes_status(void) { bool ret = false; mutex_lock(&stats_sizes_lock); if (stats_sizes_hist != NULL) ret = true; mutex_unlock(&stats_sizes_lock); return ret; } void item_stats_sizes_init(void) { if (stats_sizes_hist != NULL) return; stats_sizes_buckets = settings.item_size_max / 32 + 1; stats_sizes_hist = calloc(stats_sizes_buckets, sizeof(int)); stats_sizes_cas_min = (settings.use_cas) ? get_cas_id() : 0; } void item_stats_sizes_enable(ADD_STAT add_stats, void *c) { mutex_lock(&stats_sizes_lock); if (!settings.use_cas) { APPEND_STAT("sizes_status", "error", ""); APPEND_STAT("sizes_error", "cas_support_disabled", ""); } else if (stats_sizes_hist == NULL) { item_stats_sizes_init(); if (stats_sizes_hist != NULL) { APPEND_STAT("sizes_status", "enabled", ""); } else { APPEND_STAT("sizes_status", "error", ""); APPEND_STAT("sizes_error", "no_memory", ""); } } else { APPEND_STAT("sizes_status", "enabled", ""); } mutex_unlock(&stats_sizes_lock); } void item_stats_sizes_disable(ADD_STAT add_stats, void *c) { mutex_lock(&stats_sizes_lock); if (stats_sizes_hist != NULL) { free(stats_sizes_hist); stats_sizes_hist = NULL; } APPEND_STAT("sizes_status", "disabled", ""); mutex_unlock(&stats_sizes_lock); } void item_stats_sizes_add(item *it) { if (stats_sizes_hist == NULL || stats_sizes_cas_min > ITEM_get_cas(it)) return; int ntotal = ITEM_ntotal(it); int bucket = ntotal / 32; if ((ntotal % 32) != 0) bucket++; if (bucket < stats_sizes_buckets) stats_sizes_hist[bucket]++; } /* I think there's no way for this to be accurate without using the CAS value. * Since items getting their time value bumped will pass this validation. */ void item_stats_sizes_remove(item *it) { if (stats_sizes_hist == NULL || stats_sizes_cas_min > ITEM_get_cas(it)) return; int ntotal = ITEM_ntotal(it); int bucket = ntotal / 32; if ((ntotal % 32) != 0) bucket++; if (bucket < stats_sizes_buckets) stats_sizes_hist[bucket]--; } /** dumps out a list of objects of each size, with granularity of 32 bytes */ /*@null@*/ /* Locks are correct based on a technicality. Holds LRU lock while doing the * work, so items can't go invalid, and it's only looking at header sizes * which don't change. */ void item_stats_sizes(ADD_STAT add_stats, void *c) { mutex_lock(&stats_sizes_lock); if (stats_sizes_hist != NULL) { int i; for (i = 0; i < stats_sizes_buckets; i++) { if (stats_sizes_hist[i] != 0) { char key[12]; snprintf(key, sizeof(key), "%d", i * 32); APPEND_STAT(key, "%u", stats_sizes_hist[i]); } } } else { APPEND_STAT("sizes_status", "disabled", ""); } add_stats(NULL, 0, NULL, 0, c); mutex_unlock(&stats_sizes_lock); } /** wrapper around assoc_find which does the lazy expiration logic */ item *do_item_get(const char *key, const size_t nkey, const uint32_t hv, conn *c, const bool do_update) { item *it = assoc_find(key, nkey, hv); if (it != NULL) { refcount_incr(it); /* Optimization for slab reassignment. prevents popular items from * jamming in busy wait. Can only do this here to satisfy lock order * of item_lock, slabs_lock. */ /* This was made unsafe by removal of the cache_lock: * slab_rebalance_signal and slab_rebal.* are modified in a separate * thread under slabs_lock. If slab_rebalance_signal = 1, slab_start = * NULL (0), but slab_end is still equal to some value, this would end * up unlinking every item fetched. * This is either an acceptable loss, or if slab_rebalance_signal is * true, slab_start/slab_end should be put behind the slabs_lock. * Which would cause a huge potential slowdown. * Could also use a specific lock for slab_rebal.* and * slab_rebalance_signal (shorter lock?) */ /*if (slab_rebalance_signal && ((void *)it >= slab_rebal.slab_start && (void *)it < slab_rebal.slab_end)) { do_item_unlink(it, hv); do_item_remove(it); it = NULL; }*/ } int was_found = 0; if (settings.verbose > 2) { int ii; if (it == NULL) { fprintf(stderr, "> NOT FOUND "); } else { fprintf(stderr, "> FOUND KEY "); } for (ii = 0; ii < nkey; ++ii) { fprintf(stderr, "%c", key[ii]); } } if (it != NULL) { was_found = 1; if (item_is_flushed(it)) { do_item_unlink(it, hv); STORAGE_delete(c->thread->storage, it); do_item_remove(it); it = NULL; pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.get_flushed++; pthread_mutex_unlock(&c->thread->stats.mutex); if (settings.verbose > 2) { fprintf(stderr, " -nuked by flush"); } was_found = 2; } else if (it->exptime != 0 && it->exptime <= current_time) { do_item_unlink(it, hv); STORAGE_delete(c->thread->storage, it); do_item_remove(it); it = NULL; pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.get_expired++; pthread_mutex_unlock(&c->thread->stats.mutex); if (settings.verbose > 2) { fprintf(stderr, " -nuked by expire"); } was_found = 3; } else { if (do_update) { do_item_bump(c, it, hv); } DEBUG_REFCNT(it, '+'); } } if (settings.verbose > 2) fprintf(stderr, "\n"); /* For now this is in addition to the above verbose logging. */ LOGGER_LOG(c->thread->l, LOG_FETCHERS, LOGGER_ITEM_GET, NULL, was_found, key, nkey, (it) ? ITEM_clsid(it) : 0, c->sfd); return it; } // Requires lock held for item. // Split out of do_item_get() to allow mget functions to look through header // data before losing state modified via the bump function. void do_item_bump(conn *c, item *it, const uint32_t hv) { /* We update the hit markers only during fetches. * An item needs to be hit twice overall to be considered * ACTIVE, but only needs a single hit to maintain activity * afterward. * FETCHED tells if an item has ever been active. */ if (settings.lru_segmented) { if ((it->it_flags & ITEM_ACTIVE) == 0) { if ((it->it_flags & ITEM_FETCHED) == 0) { it->it_flags |= ITEM_FETCHED; } else { it->it_flags |= ITEM_ACTIVE; if (ITEM_lruid(it) != COLD_LRU) { it->time = current_time; // only need to bump time. } else if (!lru_bump_async(c->thread->lru_bump_buf, it, hv)) { // add flag before async bump to avoid race. it->it_flags &= ~ITEM_ACTIVE; } } } } else { it->it_flags |= ITEM_FETCHED; do_item_update(it); } } item *do_item_touch(const char *key, size_t nkey, uint32_t exptime, const uint32_t hv, conn *c) { item *it = do_item_get(key, nkey, hv, c, DO_UPDATE); if (it != NULL) { it->exptime = exptime; } return it; } /*** LRU MAINTENANCE THREAD ***/ /* Returns number of items remove, expired, or evicted. * Callable from worker threads or the LRU maintainer thread */ int lru_pull_tail(const int orig_id, const int cur_lru, const uint64_t total_bytes, const uint8_t flags, const rel_time_t max_age, struct lru_pull_tail_return *ret_it) { item *it = NULL; int id = orig_id; int removed = 0; if (id == 0) return 0; int tries = 5; item *search; item *next_it; void *hold_lock = NULL; unsigned int move_to_lru = 0; uint64_t limit = 0; id |= cur_lru; pthread_mutex_lock(&lru_locks[id]); search = tails[id]; /* We walk up *only* for locked items, and if bottom is expired. */ for (; tries > 0 && search != NULL; tries--, search=next_it) { /* we might relink search mid-loop, so search->prev isn't reliable */ next_it = search->prev; if (search->nbytes == 0 && search->nkey == 0 && search->it_flags == 1) { /* We are a crawler, ignore it. */ if (flags & LRU_PULL_CRAWL_BLOCKS) { pthread_mutex_unlock(&lru_locks[id]); return 0; } tries++; continue; } uint32_t hv = hash(ITEM_key(search), search->nkey); /* Attempt to hash item lock the "search" item. If locked, no * other callers can incr the refcount. Also skip ourselves. */ if ((hold_lock = item_trylock(hv)) == NULL) continue; /* Now see if the item is refcount locked */ if (refcount_incr(search) != 2) { /* Note pathological case with ref'ed items in tail. * Can still unlink the item, but it won't be reusable yet */ itemstats[id].lrutail_reflocked++; /* In case of refcount leaks, enable for quick workaround. */ /* WARNING: This can cause terrible corruption */ if (settings.tail_repair_time && search->time + settings.tail_repair_time < current_time) { itemstats[id].tailrepairs++; search->refcount = 1; /* This will call item_remove -> item_free since refcnt is 1 */ STORAGE_delete(ext_storage, search); do_item_unlink_nolock(search, hv); item_trylock_unlock(hold_lock); continue; } } /* Expired or flushed */ if ((search->exptime != 0 && search->exptime < current_time) || item_is_flushed(search)) { itemstats[id].reclaimed++; if ((search->it_flags & ITEM_FETCHED) == 0) { itemstats[id].expired_unfetched++; } /* refcnt 2 -> 1 */ do_item_unlink_nolock(search, hv); STORAGE_delete(ext_storage, search); /* refcnt 1 -> 0 -> item_free */ do_item_remove(search); item_trylock_unlock(hold_lock); removed++; /* If all we're finding are expired, can keep going */ continue; } /* If we're HOT_LRU or WARM_LRU and over size limit, send to COLD_LRU. * If we're COLD_LRU, send to WARM_LRU unless we need to evict */ switch (cur_lru) { case HOT_LRU: limit = total_bytes * settings.hot_lru_pct / 100; case WARM_LRU: if (limit == 0) limit = total_bytes * settings.warm_lru_pct / 100; /* Rescue ACTIVE items aggressively */ if ((search->it_flags & ITEM_ACTIVE) != 0) { search->it_flags &= ~ITEM_ACTIVE; removed++; if (cur_lru == WARM_LRU) { itemstats[id].moves_within_lru++; do_item_unlink_q(search); do_item_link_q(search); do_item_remove(search); item_trylock_unlock(hold_lock); } else { /* Active HOT_LRU items flow to WARM */ itemstats[id].moves_to_warm++; move_to_lru = WARM_LRU; do_item_unlink_q(search); it = search; } } else if (sizes_bytes[id] > limit || current_time - search->time > max_age) { itemstats[id].moves_to_cold++; move_to_lru = COLD_LRU; do_item_unlink_q(search); it = search; removed++; break; } else { /* Don't want to move to COLD, not active, bail out */ it = search; } break; case COLD_LRU: it = search; /* No matter what, we're stopping */ if (flags & LRU_PULL_EVICT) { if (settings.evict_to_free == 0) { /* Don't think we need a counter for this. It'll OOM. */ break; } itemstats[id].evicted++; itemstats[id].evicted_time = current_time - search->time; if (search->exptime != 0) itemstats[id].evicted_nonzero++; if ((search->it_flags & ITEM_FETCHED) == 0) { itemstats[id].evicted_unfetched++; } if ((search->it_flags & ITEM_ACTIVE)) { itemstats[id].evicted_active++; } LOGGER_LOG(NULL, LOG_EVICTIONS, LOGGER_EVICTION, search); STORAGE_delete(ext_storage, search); do_item_unlink_nolock(search, hv); removed++; if (settings.slab_automove == 2) { slabs_reassign(-1, orig_id); } } else if (flags & LRU_PULL_RETURN_ITEM) { /* Keep a reference to this item and return it. */ ret_it->it = it; ret_it->hv = hv; } else if ((search->it_flags & ITEM_ACTIVE) != 0 && settings.lru_segmented) { itemstats[id].moves_to_warm++; search->it_flags &= ~ITEM_ACTIVE; move_to_lru = WARM_LRU; do_item_unlink_q(search); removed++; } break; case TEMP_LRU: it = search; /* Kill the loop. Parent only interested in reclaims */ break; } if (it != NULL) break; } pthread_mutex_unlock(&lru_locks[id]); if (it != NULL) { if (move_to_lru) { it->slabs_clsid = ITEM_clsid(it); it->slabs_clsid |= move_to_lru; item_link_q(it); } if ((flags & LRU_PULL_RETURN_ITEM) == 0) { do_item_remove(it); item_trylock_unlock(hold_lock); } } return removed; } /* TODO: Third place this code needs to be deduped */ static void lru_bump_buf_link_q(lru_bump_buf *b) { pthread_mutex_lock(&bump_buf_lock); assert(b != bump_buf_head); b->prev = 0; b->next = bump_buf_head; if (b->next) b->next->prev = b; bump_buf_head = b; if (bump_buf_tail == 0) bump_buf_tail = b; pthread_mutex_unlock(&bump_buf_lock); return; } void *item_lru_bump_buf_create(void) { lru_bump_buf *b = calloc(1, sizeof(lru_bump_buf)); if (b == NULL) { return NULL; } b->buf = bipbuf_new(sizeof(lru_bump_entry) * LRU_BUMP_BUF_SIZE); if (b->buf == NULL) { free(b); return NULL; } pthread_mutex_init(&b->mutex, NULL); lru_bump_buf_link_q(b); return b; } static bool lru_bump_async(lru_bump_buf *b, item *it, uint32_t hv) { bool ret = true; refcount_incr(it); pthread_mutex_lock(&b->mutex); lru_bump_entry *be = (lru_bump_entry *) bipbuf_request(b->buf, sizeof(lru_bump_entry)); if (be != NULL) { be->it = it; be->hv = hv; if (bipbuf_push(b->buf, sizeof(lru_bump_entry)) == 0) { ret = false; b->dropped++; } } else { ret = false; b->dropped++; } if (!ret) { refcount_decr(it); } pthread_mutex_unlock(&b->mutex); return ret; } /* TODO: Might be worth a micro-optimization of having bump buffers link * themselves back into the central queue when queue goes from zero to * non-zero, then remove from list if zero more than N times. * If very few hits on cold this would avoid extra memory barriers from LRU * maintainer thread. If many hits, they'll just stay in the list. */ static bool lru_maintainer_bumps(void) { lru_bump_buf *b; lru_bump_entry *be; unsigned int size; unsigned int todo; bool bumped = false; pthread_mutex_lock(&bump_buf_lock); for (b = bump_buf_head; b != NULL; b=b->next) { pthread_mutex_lock(&b->mutex); be = (lru_bump_entry *) bipbuf_peek_all(b->buf, &size); pthread_mutex_unlock(&b->mutex); if (be == NULL) { continue; } todo = size; bumped = true; while (todo) { item_lock(be->hv); do_item_update(be->it); do_item_remove(be->it); item_unlock(be->hv); be++; todo -= sizeof(lru_bump_entry); } pthread_mutex_lock(&b->mutex); be = (lru_bump_entry *) bipbuf_poll(b->buf, size); pthread_mutex_unlock(&b->mutex); } pthread_mutex_unlock(&bump_buf_lock); return bumped; } static uint64_t lru_total_bumps_dropped(void) { uint64_t total = 0; lru_bump_buf *b; pthread_mutex_lock(&bump_buf_lock); for (b = bump_buf_head; b != NULL; b=b->next) { pthread_mutex_lock(&b->mutex); total += b->dropped; pthread_mutex_unlock(&b->mutex); } pthread_mutex_unlock(&bump_buf_lock); return total; } /* Loop up to N times: * If too many items are in HOT_LRU, push to COLD_LRU * If too many items are in WARM_LRU, push to COLD_LRU * If too many items are in COLD_LRU, poke COLD_LRU tail * 1000 loops with 1ms min sleep gives us under 1m items shifted/sec. The * locks can't handle much more than that. Leaving a TODO for how to * autoadjust in the future. */ static int lru_maintainer_juggle(const int slabs_clsid) { int i; int did_moves = 0; uint64_t total_bytes = 0; unsigned int chunks_perslab = 0; //unsigned int chunks_free = 0; /* TODO: if free_chunks below high watermark, increase aggressiveness */ slabs_available_chunks(slabs_clsid, NULL, &chunks_perslab); if (settings.temp_lru) { /* Only looking for reclaims. Run before we size the LRU. */ for (i = 0; i < 500; i++) { if (lru_pull_tail(slabs_clsid, TEMP_LRU, 0, 0, 0, NULL) <= 0) { break; } else { did_moves++; } } } rel_time_t cold_age = 0; rel_time_t hot_age = 0; rel_time_t warm_age = 0; /* If LRU is in flat mode, force items to drain into COLD via max age of 0 */ if (settings.lru_segmented) { pthread_mutex_lock(&lru_locks[slabs_clsid|COLD_LRU]); if (tails[slabs_clsid|COLD_LRU]) { cold_age = current_time - tails[slabs_clsid|COLD_LRU]->time; } // Also build up total_bytes for the classes. total_bytes += sizes_bytes[slabs_clsid|COLD_LRU]; pthread_mutex_unlock(&lru_locks[slabs_clsid|COLD_LRU]); hot_age = cold_age * settings.hot_max_factor; warm_age = cold_age * settings.warm_max_factor; // total_bytes doesn't have to be exact. cache it for the juggles. pthread_mutex_lock(&lru_locks[slabs_clsid|HOT_LRU]); total_bytes += sizes_bytes[slabs_clsid|HOT_LRU]; pthread_mutex_unlock(&lru_locks[slabs_clsid|HOT_LRU]); pthread_mutex_lock(&lru_locks[slabs_clsid|WARM_LRU]); total_bytes += sizes_bytes[slabs_clsid|WARM_LRU]; pthread_mutex_unlock(&lru_locks[slabs_clsid|WARM_LRU]); } /* Juggle HOT/WARM up to N times */ for (i = 0; i < 500; i++) { int do_more = 0; if (lru_pull_tail(slabs_clsid, HOT_LRU, total_bytes, LRU_PULL_CRAWL_BLOCKS, hot_age, NULL) || lru_pull_tail(slabs_clsid, WARM_LRU, total_bytes, LRU_PULL_CRAWL_BLOCKS, warm_age, NULL)) { do_more++; } if (settings.lru_segmented) { do_more += lru_pull_tail(slabs_clsid, COLD_LRU, total_bytes, LRU_PULL_CRAWL_BLOCKS, 0, NULL); } if (do_more == 0) break; did_moves++; } return did_moves; } /* Will crawl all slab classes a minimum of once per hour */ #define MAX_MAINTCRAWL_WAIT 60 * 60 /* Hoping user input will improve this function. This is all a wild guess. * Operation: Kicks crawler for each slab id. Crawlers take some statistics as * to items with nonzero expirations. It then buckets how many items will * expire per minute for the next hour. * This function checks the results of a run, and if it things more than 1% of * expirable objects are ready to go, kick the crawler again to reap. * It will also kick the crawler once per minute regardless, waiting a minute * longer for each time it has no work to do, up to an hour wait time. * The latter is to avoid newly started daemons from waiting too long before * retrying a crawl. */ static void lru_maintainer_crawler_check(struct crawler_expired_data *cdata, logger *l) { int i; static rel_time_t next_crawls[POWER_LARGEST]; static rel_time_t next_crawl_wait[POWER_LARGEST]; uint8_t todo[POWER_LARGEST]; memset(todo, 0, sizeof(uint8_t) * POWER_LARGEST); bool do_run = false; unsigned int tocrawl_limit = 0; // TODO: If not segmented LRU, skip non-cold for (i = POWER_SMALLEST; i < POWER_LARGEST; i++) { crawlerstats_t *s = &cdata->crawlerstats[i]; /* We've not successfully kicked off a crawl yet. */ if (s->run_complete) { char *lru_name = "na"; pthread_mutex_lock(&cdata->lock); int x; /* Should we crawl again? */ uint64_t possible_reclaims = s->seen - s->noexp; uint64_t available_reclaims = 0; /* Need to think we can free at least 1% of the items before * crawling. */ /* FIXME: Configurable? */ uint64_t low_watermark = (possible_reclaims / 100) + 1; rel_time_t since_run = current_time - s->end_time; /* Don't bother if the payoff is too low. */ for (x = 0; x < 60; x++) { available_reclaims += s->histo[x]; if (available_reclaims > low_watermark) { if (next_crawl_wait[i] < (x * 60)) { next_crawl_wait[i] += 60; } else if (next_crawl_wait[i] >= 60) { next_crawl_wait[i] -= 60; } break; } } if (available_reclaims == 0) { next_crawl_wait[i] += 60; } if (next_crawl_wait[i] > MAX_MAINTCRAWL_WAIT) { next_crawl_wait[i] = MAX_MAINTCRAWL_WAIT; } next_crawls[i] = current_time + next_crawl_wait[i] + 5; switch (GET_LRU(i)) { case HOT_LRU: lru_name = "hot"; break; case WARM_LRU: lru_name = "warm"; break; case COLD_LRU: lru_name = "cold"; break; case TEMP_LRU: lru_name = "temp"; break; } LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_CRAWLER_STATUS, NULL, CLEAR_LRU(i), lru_name, (unsigned long long)low_watermark, (unsigned long long)available_reclaims, (unsigned int)since_run, next_crawls[i] - current_time, s->end_time - s->start_time, s->seen, s->reclaimed); // Got our calculation, avoid running until next actual run. s->run_complete = false; pthread_mutex_unlock(&cdata->lock); } if (current_time > next_crawls[i]) { pthread_mutex_lock(&lru_locks[i]); if (sizes[i] > tocrawl_limit) { tocrawl_limit = sizes[i]; } pthread_mutex_unlock(&lru_locks[i]); todo[i] = 1; do_run = true; next_crawls[i] = current_time + 5; // minimum retry wait. } } if (do_run) { if (settings.lru_crawler_tocrawl && settings.lru_crawler_tocrawl < tocrawl_limit) { tocrawl_limit = settings.lru_crawler_tocrawl; } lru_crawler_start(todo, tocrawl_limit, CRAWLER_AUTOEXPIRE, cdata, NULL, 0); } } slab_automove_reg_t slab_automove_default = { .init = slab_automove_init, .free = slab_automove_free, .run = slab_automove_run }; #ifdef EXTSTORE slab_automove_reg_t slab_automove_extstore = { .init = slab_automove_extstore_init, .free = slab_automove_extstore_free, .run = slab_automove_extstore_run }; #endif static pthread_t lru_maintainer_tid; #define MAX_LRU_MAINTAINER_SLEEP 1000000 #define MIN_LRU_MAINTAINER_SLEEP 1000 static void *lru_maintainer_thread(void *arg) { slab_automove_reg_t *sam = &slab_automove_default; #ifdef EXTSTORE void *storage = arg; if (storage != NULL) sam = &slab_automove_extstore; #endif int i; useconds_t to_sleep = MIN_LRU_MAINTAINER_SLEEP; useconds_t last_sleep = MIN_LRU_MAINTAINER_SLEEP; rel_time_t last_crawler_check = 0; rel_time_t last_automove_check = 0; useconds_t next_juggles[MAX_NUMBER_OF_SLAB_CLASSES] = {0}; useconds_t backoff_juggles[MAX_NUMBER_OF_SLAB_CLASSES] = {0}; struct crawler_expired_data *cdata = calloc(1, sizeof(struct crawler_expired_data)); if (cdata == NULL) { fprintf(stderr, "Failed to allocate crawler data for LRU maintainer thread\n"); abort(); } pthread_mutex_init(&cdata->lock, NULL); cdata->crawl_complete = true; // kick off the crawler. logger *l = logger_create(); if (l == NULL) { fprintf(stderr, "Failed to allocate logger for LRU maintainer thread\n"); abort(); } double last_ratio = settings.slab_automove_ratio; void *am = sam->init(&settings); pthread_mutex_lock(&lru_maintainer_lock); if (settings.verbose > 2) fprintf(stderr, "Starting LRU maintainer background thread\n"); while (do_run_lru_maintainer_thread) { pthread_mutex_unlock(&lru_maintainer_lock); if (to_sleep) usleep(to_sleep); pthread_mutex_lock(&lru_maintainer_lock); /* A sleep of zero counts as a minimum of a 1ms wait */ last_sleep = to_sleep > 1000 ? to_sleep : 1000; to_sleep = MAX_LRU_MAINTAINER_SLEEP; STATS_LOCK(); stats.lru_maintainer_juggles++; STATS_UNLOCK(); /* Each slab class gets its own sleep to avoid hammering locks */ for (i = POWER_SMALLEST; i < MAX_NUMBER_OF_SLAB_CLASSES; i++) { next_juggles[i] = next_juggles[i] > last_sleep ? next_juggles[i] - last_sleep : 0; if (next_juggles[i] > 0) { // Sleep the thread just for the minimum amount (or not at all) if (next_juggles[i] < to_sleep) to_sleep = next_juggles[i]; continue; } int did_moves = lru_maintainer_juggle(i); if (did_moves == 0) { if (backoff_juggles[i] != 0) { backoff_juggles[i] += backoff_juggles[i] / 8; } else { backoff_juggles[i] = MIN_LRU_MAINTAINER_SLEEP; } if (backoff_juggles[i] > MAX_LRU_MAINTAINER_SLEEP) backoff_juggles[i] = MAX_LRU_MAINTAINER_SLEEP; } else if (backoff_juggles[i] > 0) { backoff_juggles[i] /= 2; if (backoff_juggles[i] < MIN_LRU_MAINTAINER_SLEEP) { backoff_juggles[i] = 0; } } next_juggles[i] = backoff_juggles[i]; if (next_juggles[i] < to_sleep) to_sleep = next_juggles[i]; } /* Minimize the sleep if we had async LRU bumps to process */ if (settings.lru_segmented && lru_maintainer_bumps() && to_sleep > 1000) { to_sleep = 1000; } /* Once per second at most */ if (settings.lru_crawler && last_crawler_check != current_time) { lru_maintainer_crawler_check(cdata, l); last_crawler_check = current_time; } if (settings.slab_automove == 1 && last_automove_check != current_time) { if (last_ratio != settings.slab_automove_ratio) { sam->free(am); am = sam->init(&settings); last_ratio = settings.slab_automove_ratio; } int src, dst; sam->run(am, &src, &dst); if (src != -1 && dst != -1) { slabs_reassign(src, dst); LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_SLAB_MOVE, NULL, src, dst); } // dst == 0 means reclaim to global pool, be more aggressive if (dst != 0) { last_automove_check = current_time; } else if (dst == 0) { // also ensure we minimize the thread sleep to_sleep = 1000; } } } pthread_mutex_unlock(&lru_maintainer_lock); sam->free(am); // LRU crawler *must* be stopped. free(cdata); if (settings.verbose > 2) fprintf(stderr, "LRU maintainer thread stopping\n"); return NULL; } int stop_lru_maintainer_thread(void) { int ret; pthread_mutex_lock(&lru_maintainer_lock); /* LRU thread is a sleep loop, will die on its own */ do_run_lru_maintainer_thread = 0; pthread_mutex_unlock(&lru_maintainer_lock); if ((ret = pthread_join(lru_maintainer_tid, NULL)) != 0) { fprintf(stderr, "Failed to stop LRU maintainer thread: %s\n", strerror(ret)); return -1; } settings.lru_maintainer_thread = false; return 0; } int start_lru_maintainer_thread(void *arg) { int ret; pthread_mutex_lock(&lru_maintainer_lock); do_run_lru_maintainer_thread = 1; settings.lru_maintainer_thread = true; if ((ret = pthread_create(&lru_maintainer_tid, NULL, lru_maintainer_thread, arg)) != 0) { fprintf(stderr, "Can't create LRU maintainer thread: %s\n", strerror(ret)); pthread_mutex_unlock(&lru_maintainer_lock); return -1; } pthread_mutex_unlock(&lru_maintainer_lock); return 0; } /* If we hold this lock, crawler can't wake up or move */ void lru_maintainer_pause(void) { pthread_mutex_lock(&lru_maintainer_lock); } void lru_maintainer_resume(void) { pthread_mutex_unlock(&lru_maintainer_lock); } int init_lru_maintainer(void) { lru_maintainer_initialized = 1; return 0; } /* Tail linkers and crawler for the LRU crawler. */ void do_item_linktail_q(item *it) { /* item is the new tail */ item **head, **tail; assert(it->it_flags == 1); assert(it->nbytes == 0); head = &heads[it->slabs_clsid]; tail = &tails[it->slabs_clsid]; //assert(*tail != 0); assert(it != *tail); assert((*head && *tail) || (*head == 0 && *tail == 0)); it->prev = *tail; it->next = 0; if (it->prev) { assert(it->prev->next == 0); it->prev->next = it; } *tail = it; if (*head == 0) *head = it; return; } void do_item_unlinktail_q(item *it) { item **head, **tail; head = &heads[it->slabs_clsid]; tail = &tails[it->slabs_clsid]; if (*head == it) { assert(it->prev == 0); *head = it->next; } if (*tail == it) { assert(it->next == 0); *tail = it->prev; } assert(it->next != it); assert(it->prev != it); if (it->next) it->next->prev = it->prev; if (it->prev) it->prev->next = it->next; return; } /* This is too convoluted, but it's a difficult shuffle. Try to rewrite it * more clearly. */ item *do_item_crawl_q(item *it) { item **head, **tail; assert(it->it_flags == 1); assert(it->nbytes == 0); head = &heads[it->slabs_clsid]; tail = &tails[it->slabs_clsid]; /* We've hit the head, pop off */ if (it->prev == 0) { assert(*head == it); if (it->next) { *head = it->next; assert(it->next->prev == it); it->next->prev = 0; } return NULL; /* Done */ } /* Swing ourselves in front of the next item */ /* NB: If there is a prev, we can't be the head */ assert(it->prev != it); if (it->prev) { if (*head == it->prev) { /* Prev was the head, now we're the head */ *head = it; } if (*tail == it) { /* We are the tail, now they are the tail */ *tail = it->prev; } assert(it->next != it); if (it->next) { assert(it->prev->next == it); it->prev->next = it->next; it->next->prev = it->prev; } else { /* Tail. Move this above? */ it->prev->next = 0; } /* prev->prev's next is it->prev */ it->next = it->prev; it->prev = it->next->prev; it->next->prev = it; /* New it->prev now, if we're not at the head. */ if (it->prev) { it->prev->next = it; } } assert(it->next != it); assert(it->prev != it); return it->next; /* success */ } memcached-1.5.22/hash.c0000664000175000017500000000103313615376577011542 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #include "memcached.h" #include "jenkins_hash.h" #include "murmur3_hash.h" int hash_init(enum hashfunc_type type) { switch(type) { case JENKINS_HASH: hash = jenkins_hash; settings.hash_algorithm = "jenkins"; break; case MURMUR3_HASH: hash = MurmurHash3_x86_32; settings.hash_algorithm = "murmur3"; break; default: return -1; } return 0; } memcached-1.5.22/murmur3_hash.c0000664000175000017500000000541213461170555013224 00000000000000//----------------------------------------------------------------------------- // MurmurHash3 was written by Austin Appleby, and is placed in the public // domain. The author hereby disclaims copyright to this source code. // Note - The x86 and x64 versions do _not_ produce the same results, as the // algorithms are optimized for their respective platforms. You can still // compile and run any of them on any platform, but your performance with the // non-native version will be less than optimal. #include "murmur3_hash.h" //----------------------------------------------------------------------------- // Platform-specific functions and macros // Microsoft Visual Studio #if defined(_MSC_VER) #define FORCE_INLINE __forceinline #include #define ROTL32(x,y) _rotl(x,y) #define BIG_CONSTANT(x) (x) // Other compilers #else // defined(_MSC_VER) #define FORCE_INLINE inline __attribute__((always_inline)) static inline uint32_t rotl32 ( uint32_t x, int8_t r ) { return (x << r) | (x >> (32 - r)); } #define ROTL32(x,y) rotl32(x,y) #define BIG_CONSTANT(x) (x##LLU) #endif // !defined(_MSC_VER) //----------------------------------------------------------------------------- // Block read - if your platform needs to do endian-swapping or can only // handle aligned reads, do the conversion here static FORCE_INLINE uint32_t getblock32 ( const uint32_t * p, int i ) { return p[i]; } //----------------------------------------------------------------------------- // Finalization mix - force all bits of a hash block to avalanche static FORCE_INLINE uint32_t fmix32 ( uint32_t h ) { h ^= h >> 16; h *= 0x85ebca6b; h ^= h >> 13; h *= 0xc2b2ae35; h ^= h >> 16; return h; } //----------------------------------------------------------------------------- /* Definition modified slightly from the public domain interface (no seed + * return value */ uint32_t MurmurHash3_x86_32 ( const void * key, size_t length) { const uint8_t * data = (const uint8_t*)key; const int nblocks = length / 4; uint32_t h1 = 0; uint32_t c1 = 0xcc9e2d51; uint32_t c2 = 0x1b873593; //---------- // body const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); for(int i = -nblocks; i; i++) { uint32_t k1 = getblock32(blocks,i); k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; h1 = ROTL32(h1,13); h1 = h1*5+0xe6546b64; } //---------- // tail const uint8_t * tail = (const uint8_t*)(data + nblocks*4); uint32_t k1 = 0; switch(length & 3) { case 3: k1 ^= tail[2] << 16; case 2: k1 ^= tail[1] << 8; case 1: k1 ^= tail[0]; k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; }; //---------- // finalization h1 ^= length; h1 = fmix32(h1); //*(uint32_t*)out = h1; return h1; } memcached-1.5.22/tls.h0000664000175000017500000000054413575022500011407 00000000000000#ifndef TLS_H #define TLS_H void SSL_LOCK(void); void SSL_UNLOCK(void); ssize_t ssl_read(conn *c, void *buf, size_t count); ssize_t ssl_sendmsg(conn *c, struct msghdr *msg, int flags); ssize_t ssl_write(conn *c, void *buf, size_t count); int ssl_init(void); bool refresh_certs(char **errmsg); void ssl_callback(const SSL *s, int where, int ret); #endif memcached-1.5.22/config.sub0000755000175000017500000010645013244306071012421 00000000000000#! /bin/sh # Configuration validation subroutine script. # Copyright 1992-2018 Free Software Foundation, Inc. timestamp='2018-02-22' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, see . # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that # program. This Exception is an additional permission under section 7 # of the GNU General Public License, version 3 ("GPLv3"). # Please send patches to . # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. # If it is invalid, we print an error message on stderr and exit with code 1. # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: # https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases # that are meaningful with *any* GNU software. # Each package is responsible for reporting which valid configurations # it does not support. The user should be able to distinguish # a failure to support a valid configuration from a meaningless # configuration. # The goal of this file is to map all the various variations of a given # machine specification into a single specification in the form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM # or in some cases, the newer four-part form: # CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM # It is wrong to echo any other type of specification. me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit Report bugs and patches to ." version="\ GNU config.sub ($timestamp) Copyright 1992-2018 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" Try \`$me --help' for more information." # Parse command line while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) echo "$timestamp" ; exit ;; --version | -v ) echo "$version" ; exit ;; --help | --h* | -h ) echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. break ;; -* ) echo "$me: invalid option $1$help" exit 1 ;; *local*) # First pass through any local machine types. echo "$1" exit ;; * ) break ;; esac done case $# in 0) echo "$me: missing argument$help" >&2 exit 1;; 1) ;; *) echo "$me: too many arguments$help" >&2 exit 1;; esac # Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). # Here we must recognize all the valid KERNEL-OS combinations. maybe_os=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ kopensolaris*-gnu* | cloudabi*-eabi* | \ storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` ;; android-linux) os=-linux-android basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown ;; *) basic_machine=`echo "$1" | sed 's/-[^-]*$//'` if [ "$basic_machine" != "$1" ] then os=`echo "$1" | sed 's/.*-/-/'` else os=; fi ;; esac ### Let's recognize common machines as not being operating systems so ### that things like config.sub decstation-3100 work. We also ### recognize some manufacturers as not being operating systems, so we ### can provide default operating systems below. case $os in -sun*os*) # Prevent following clause from handling this invalid input. ;; -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ -apple | -axis | -knuth | -cray | -microblaze*) os= basic_machine=$1 ;; -bluegene*) os=-cnk ;; -sim | -cisco | -oki | -wec | -winbond) os= basic_machine=$1 ;; -scout) ;; -wrs) os=-vxworks basic_machine=$1 ;; -chorusos*) os=-chorusos basic_machine=$1 ;; -chorusrdb) os=-chorusrdb basic_machine=$1 ;; -hiux*) os=-hiuxwe2 ;; -sco6) os=-sco5v6 basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco5) os=-sco3.2v5 basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco4) os=-sco3.2v4 basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco3.2.[4-9]*) os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco3.2v[4-9]*) # Don't forget version if it is 3.2v4 or newer. basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco5v6*) # Don't forget version if it is 3.2v4 or newer. basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco*) os=-sco3.2v2 basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -udk*) basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -isc) os=-isc2.2 basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -clix*) basic_machine=clipper-intergraph ;; -isc*) basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -lynx*178) os=-lynxos178 ;; -lynx*5) os=-lynxos5 ;; -lynx*) os=-lynxos ;; -ptx*) basic_machine=`echo "$1" | sed -e 's/86-.*/86-sequent/'` ;; -psos*) os=-psos ;; -mint | -mint[0-9]*) basic_machine=m68k-atari os=-mint ;; esac # Decode aliases for certain CPU-COMPANY combinations. case $basic_machine in # Recognize the basic CPU types without company name. # Some are omitted here because they have special meanings below. 1750a | 580 \ | a29k \ | aarch64 | aarch64_be \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ | am33_2.0 \ | arc | arceb \ | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ | avr | avr32 \ | ba \ | be32 | be64 \ | bfin \ | c4x | c8051 | clipper \ | d10v | d30v | dlx | dsp16xx \ | e2k | epiphany \ | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ | i370 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ | le32 | le64 \ | lm32 \ | m32c | m32r | m32rle | m68000 | m68k | m88k \ | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ | mips | mipsbe | mipseb | mipsel | mipsle \ | mips16 \ | mips64 | mips64el \ | mips64octeon | mips64octeonel \ | mips64orion | mips64orionel \ | mips64r5900 | mips64r5900el \ | mips64vr | mips64vrel \ | mips64vr4100 | mips64vr4100el \ | mips64vr4300 | mips64vr4300el \ | mips64vr5000 | mips64vr5000el \ | mips64vr5900 | mips64vr5900el \ | mipsisa32 | mipsisa32el \ | mipsisa32r2 | mipsisa32r2el \ | mipsisa32r6 | mipsisa32r6el \ | mipsisa64 | mipsisa64el \ | mipsisa64r2 | mipsisa64r2el \ | mipsisa64r6 | mipsisa64r6el \ | mipsisa64sb1 | mipsisa64sb1el \ | mipsisa64sr71k | mipsisa64sr71kel \ | mipsr5900 | mipsr5900el \ | mipstx39 | mipstx39el \ | mn10200 | mn10300 \ | moxie \ | mt \ | msp430 \ | nds32 | nds32le | nds32be \ | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ | open8 | or1k | or1knd | or32 \ | pdp10 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ | pru \ | pyramid \ | riscv32 | riscv64 \ | rl78 | rx \ | score \ | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ | spu \ | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ | visium \ | wasm32 \ | x86 | xc16x | xstormy16 | xtensa \ | z8k | z80) basic_machine=$basic_machine-unknown ;; c54x) basic_machine=tic54x-unknown ;; c55x) basic_machine=tic55x-unknown ;; c6x) basic_machine=tic6x-unknown ;; leon|leon[3-9]) basic_machine=sparc-$basic_machine ;; m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip) basic_machine=$basic_machine-unknown os=-none ;; m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65) ;; ms1) basic_machine=mt-unknown ;; strongarm | thumb | xscale) basic_machine=arm-unknown ;; xgate) basic_machine=$basic_machine-unknown os=-none ;; xscaleeb) basic_machine=armeb-unknown ;; xscaleel) basic_machine=armel-unknown ;; # We use `pc' rather than `unknown' # because (1) that's what they normally are, and # (2) the word "unknown" tends to confuse beginning users. i*86 | x86_64) basic_machine=$basic_machine-pc ;; # Object if more than one company name word. *-*-*) echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 exit 1 ;; # Recognize the basic CPU types with company name. 580-* \ | a29k-* \ | aarch64-* | aarch64_be-* \ | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ | avr-* | avr32-* \ | ba-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ | c8051-* | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ | e2k-* | elxsi-* \ | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ | ip2k-* | iq2000-* \ | k1om-* \ | le32-* | le64-* \ | lm32-* \ | m32c-* | m32r-* | m32rle-* \ | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ | microblaze-* | microblazeel-* \ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ | mips16-* \ | mips64-* | mips64el-* \ | mips64octeon-* | mips64octeonel-* \ | mips64orion-* | mips64orionel-* \ | mips64r5900-* | mips64r5900el-* \ | mips64vr-* | mips64vrel-* \ | mips64vr4100-* | mips64vr4100el-* \ | mips64vr4300-* | mips64vr4300el-* \ | mips64vr5000-* | mips64vr5000el-* \ | mips64vr5900-* | mips64vr5900el-* \ | mipsisa32-* | mipsisa32el-* \ | mipsisa32r2-* | mipsisa32r2el-* \ | mipsisa32r6-* | mipsisa32r6el-* \ | mipsisa64-* | mipsisa64el-* \ | mipsisa64r2-* | mipsisa64r2el-* \ | mipsisa64r6-* | mipsisa64r6el-* \ | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ | mipsr5900-* | mipsr5900el-* \ | mipstx39-* | mipstx39el-* \ | mmix-* \ | mt-* \ | msp430-* \ | nds32-* | nds32le-* | nds32be-* \ | nios-* | nios2-* | nios2eb-* | nios2el-* \ | none-* | np1-* | ns16k-* | ns32k-* \ | open8-* \ | or1k*-* \ | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ | pru-* \ | pyramid-* \ | riscv32-* | riscv64-* \ | rl78-* | romp-* | rs6000-* | rx-* \ | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ | sparclite-* \ | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ | tile*-* \ | tron-* \ | ubicom32-* \ | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ | visium-* \ | wasm32-* \ | we32k-* \ | x86-* | x86_64-* | xc16x-* | xps100-* \ | xstormy16-* | xtensa*-* \ | ymp-* \ | z8k-* | z80-*) ;; # Recognize the basic CPU types without company name, with glob match. xtensa*) basic_machine=$basic_machine-unknown ;; # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. 386bsd) basic_machine=i386-pc os=-bsd ;; 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) basic_machine=m68000-att ;; 3b*) basic_machine=we32k-att ;; a29khif) basic_machine=a29k-amd os=-udi ;; abacus) basic_machine=abacus-unknown ;; adobe68k) basic_machine=m68010-adobe os=-scout ;; alliant | fx80) basic_machine=fx80-alliant ;; altos | altos3068) basic_machine=m68k-altos ;; am29k) basic_machine=a29k-none os=-bsd ;; amd64) basic_machine=x86_64-pc ;; amd64-*) basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; amdahl) basic_machine=580-amdahl os=-sysv ;; amiga | amiga-*) basic_machine=m68k-unknown ;; amigaos | amigados) basic_machine=m68k-unknown os=-amigaos ;; amigaunix | amix) basic_machine=m68k-unknown os=-sysv4 ;; apollo68) basic_machine=m68k-apollo os=-sysv ;; apollo68bsd) basic_machine=m68k-apollo os=-bsd ;; aros) basic_machine=i386-pc os=-aros ;; asmjs) basic_machine=asmjs-unknown ;; aux) basic_machine=m68k-apple os=-aux ;; balance) basic_machine=ns32k-sequent os=-dynix ;; blackfin) basic_machine=bfin-unknown os=-linux ;; blackfin-*) basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'` os=-linux ;; bluegene*) basic_machine=powerpc-ibm os=-cnk ;; c54x-*) basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c55x-*) basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c6x-*) basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c90) basic_machine=c90-cray os=-unicos ;; cegcc) basic_machine=arm-unknown os=-cegcc ;; convex-c1) basic_machine=c1-convex os=-bsd ;; convex-c2) basic_machine=c2-convex os=-bsd ;; convex-c32) basic_machine=c32-convex os=-bsd ;; convex-c34) basic_machine=c34-convex os=-bsd ;; convex-c38) basic_machine=c38-convex os=-bsd ;; cray | j90) basic_machine=j90-cray os=-unicos ;; craynv) basic_machine=craynv-cray os=-unicosmp ;; cr16 | cr16-*) basic_machine=cr16-unknown os=-elf ;; crds | unos) basic_machine=m68k-crds ;; crisv32 | crisv32-* | etraxfs*) basic_machine=crisv32-axis ;; cris | cris-* | etrax*) basic_machine=cris-axis ;; crx) basic_machine=crx-unknown os=-elf ;; da30 | da30-*) basic_machine=m68k-da30 ;; decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) basic_machine=mips-dec ;; decsystem10* | dec10*) basic_machine=pdp10-dec os=-tops10 ;; decsystem20* | dec20*) basic_machine=pdp10-dec os=-tops20 ;; delta | 3300 | motorola-3300 | motorola-delta \ | 3300-motorola | delta-motorola) basic_machine=m68k-motorola ;; delta88) basic_machine=m88k-motorola os=-sysv3 ;; dicos) basic_machine=i686-pc os=-dicos ;; djgpp) basic_machine=i586-pc os=-msdosdjgpp ;; dpx20 | dpx20-*) basic_machine=rs6000-bull os=-bosx ;; dpx2*) basic_machine=m68k-bull os=-sysv3 ;; e500v[12]) basic_machine=powerpc-unknown os=$os"spe" ;; e500v[12]-*) basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` os=$os"spe" ;; ebmon29k) basic_machine=a29k-amd os=-ebmon ;; elxsi) basic_machine=elxsi-elxsi os=-bsd ;; encore | umax | mmax) basic_machine=ns32k-encore ;; es1800 | OSE68k | ose68k | ose | OSE) basic_machine=m68k-ericsson os=-ose ;; fx2800) basic_machine=i860-alliant ;; genix) basic_machine=ns32k-ns ;; gmicro) basic_machine=tron-gmicro os=-sysv ;; go32) basic_machine=i386-pc os=-go32 ;; h3050r* | hiux*) basic_machine=hppa1.1-hitachi os=-hiuxwe2 ;; h8300hms) basic_machine=h8300-hitachi os=-hms ;; h8300xray) basic_machine=h8300-hitachi os=-xray ;; h8500hms) basic_machine=h8500-hitachi os=-hms ;; harris) basic_machine=m88k-harris os=-sysv3 ;; hp300-*) basic_machine=m68k-hp ;; hp300bsd) basic_machine=m68k-hp os=-bsd ;; hp300hpux) basic_machine=m68k-hp os=-hpux ;; hp3k9[0-9][0-9] | hp9[0-9][0-9]) basic_machine=hppa1.0-hp ;; hp9k2[0-9][0-9] | hp9k31[0-9]) basic_machine=m68000-hp ;; hp9k3[2-9][0-9]) basic_machine=m68k-hp ;; hp9k6[0-9][0-9] | hp6[0-9][0-9]) basic_machine=hppa1.0-hp ;; hp9k7[0-79][0-9] | hp7[0-79][0-9]) basic_machine=hppa1.1-hp ;; hp9k78[0-9] | hp78[0-9]) # FIXME: really hppa2.0-hp basic_machine=hppa1.1-hp ;; hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) # FIXME: really hppa2.0-hp basic_machine=hppa1.1-hp ;; hp9k8[0-9][13679] | hp8[0-9][13679]) basic_machine=hppa1.1-hp ;; hp9k8[0-9][0-9] | hp8[0-9][0-9]) basic_machine=hppa1.0-hp ;; hppaosf) basic_machine=hppa1.1-hp os=-osf ;; hppro) basic_machine=hppa1.1-hp os=-proelf ;; i370-ibm* | ibm*) basic_machine=i370-ibm ;; i*86v32) basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` os=-sysv32 ;; i*86v4*) basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` os=-sysv4 ;; i*86v) basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` os=-sysv ;; i*86sol2) basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` os=-solaris2 ;; i386mach) basic_machine=i386-mach os=-mach ;; vsta) basic_machine=i386-unknown os=-vsta ;; iris | iris4d) basic_machine=mips-sgi case $os in -irix*) ;; *) os=-irix4 ;; esac ;; isi68 | isi) basic_machine=m68k-isi os=-sysv ;; leon-*|leon[3-9]-*) basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'` ;; m68knommu) basic_machine=m68k-unknown os=-linux ;; m68knommu-*) basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'` os=-linux ;; magnum | m3230) basic_machine=mips-mips os=-sysv ;; merlin) basic_machine=ns32k-utek os=-sysv ;; microblaze*) basic_machine=microblaze-xilinx ;; mingw64) basic_machine=x86_64-pc os=-mingw64 ;; mingw32) basic_machine=i686-pc os=-mingw32 ;; mingw32ce) basic_machine=arm-unknown os=-mingw32ce ;; miniframe) basic_machine=m68000-convergent ;; *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) basic_machine=m68k-atari os=-mint ;; mips3*-*) basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'` ;; mips3*) basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown ;; monitor) basic_machine=m68k-rom68k os=-coff ;; morphos) basic_machine=powerpc-unknown os=-morphos ;; moxiebox) basic_machine=moxie-unknown os=-moxiebox ;; msdos) basic_machine=i386-pc os=-msdos ;; ms1-*) basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'` ;; msys) basic_machine=i686-pc os=-msys ;; mvs) basic_machine=i370-ibm os=-mvs ;; nacl) basic_machine=le32-unknown os=-nacl ;; ncr3000) basic_machine=i486-ncr os=-sysv4 ;; netbsd386) basic_machine=i386-unknown os=-netbsd ;; netwinder) basic_machine=armv4l-rebel os=-linux ;; news | news700 | news800 | news900) basic_machine=m68k-sony os=-newsos ;; news1000) basic_machine=m68030-sony os=-newsos ;; news-3600 | risc-news) basic_machine=mips-sony os=-newsos ;; necv70) basic_machine=v70-nec os=-sysv ;; next | m*-next) basic_machine=m68k-next case $os in -nextstep* ) ;; -ns2*) os=-nextstep2 ;; *) os=-nextstep3 ;; esac ;; nh3000) basic_machine=m68k-harris os=-cxux ;; nh[45]000) basic_machine=m88k-harris os=-cxux ;; nindy960) basic_machine=i960-intel os=-nindy ;; mon960) basic_machine=i960-intel os=-mon960 ;; nonstopux) basic_machine=mips-compaq os=-nonstopux ;; np1) basic_machine=np1-gould ;; neo-tandem) basic_machine=neo-tandem ;; nse-tandem) basic_machine=nse-tandem ;; nsr-tandem) basic_machine=nsr-tandem ;; nsv-tandem) basic_machine=nsv-tandem ;; nsx-tandem) basic_machine=nsx-tandem ;; op50n-* | op60c-*) basic_machine=hppa1.1-oki os=-proelf ;; openrisc | openrisc-*) basic_machine=or32-unknown ;; os400) basic_machine=powerpc-ibm os=-os400 ;; OSE68000 | ose68000) basic_machine=m68000-ericsson os=-ose ;; os68k) basic_machine=m68k-none os=-os68k ;; pa-hitachi) basic_machine=hppa1.1-hitachi os=-hiuxwe2 ;; paragon) basic_machine=i860-intel os=-osf ;; parisc) basic_machine=hppa-unknown os=-linux ;; parisc-*) basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'` os=-linux ;; pbd) basic_machine=sparc-tti ;; pbb) basic_machine=m68k-tti ;; pc532 | pc532-*) basic_machine=ns32k-pc532 ;; pc98) basic_machine=i386-pc ;; pc98-*) basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentium | p5 | k5 | k6 | nexgen | viac3) basic_machine=i586-pc ;; pentiumpro | p6 | 6x86 | athlon | athlon_*) basic_machine=i686-pc ;; pentiumii | pentium2 | pentiumiii | pentium3) basic_machine=i686-pc ;; pentium4) basic_machine=i786-pc ;; pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentiumpro-* | p6-* | 6x86-* | athlon-*) basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentium4-*) basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pn) basic_machine=pn-gould ;; power) basic_machine=power-ibm ;; ppc | ppcbe) basic_machine=powerpc-unknown ;; ppc-* | ppcbe-*) basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ppcle | powerpclittle) basic_machine=powerpcle-unknown ;; ppcle-* | powerpclittle-*) basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ppc64) basic_machine=powerpc64-unknown ;; ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ppc64le | powerpc64little) basic_machine=powerpc64le-unknown ;; ppc64le-* | powerpc64little-*) basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ps2) basic_machine=i386-ibm ;; pw32) basic_machine=i586-unknown os=-pw32 ;; rdos | rdos64) basic_machine=x86_64-pc os=-rdos ;; rdos32) basic_machine=i386-pc os=-rdos ;; rom68k) basic_machine=m68k-rom68k os=-coff ;; rm[46]00) basic_machine=mips-siemens ;; rtpc | rtpc-*) basic_machine=romp-ibm ;; s390 | s390-*) basic_machine=s390-ibm ;; s390x | s390x-*) basic_machine=s390x-ibm ;; sa29200) basic_machine=a29k-amd os=-udi ;; sb1) basic_machine=mipsisa64sb1-unknown ;; sb1el) basic_machine=mipsisa64sb1el-unknown ;; sde) basic_machine=mipsisa32-sde os=-elf ;; sei) basic_machine=mips-sei os=-seiux ;; sequent) basic_machine=i386-sequent ;; sh5el) basic_machine=sh5le-unknown ;; simso-wrs) basic_machine=sparclite-wrs os=-vxworks ;; sps7) basic_machine=m68k-bull os=-sysv2 ;; spur) basic_machine=spur-unknown ;; st2000) basic_machine=m68k-tandem ;; stratus) basic_machine=i860-stratus os=-sysv4 ;; strongarm-* | thumb-*) basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; sun2) basic_machine=m68000-sun ;; sun2os3) basic_machine=m68000-sun os=-sunos3 ;; sun2os4) basic_machine=m68000-sun os=-sunos4 ;; sun3os3) basic_machine=m68k-sun os=-sunos3 ;; sun3os4) basic_machine=m68k-sun os=-sunos4 ;; sun4os3) basic_machine=sparc-sun os=-sunos3 ;; sun4os4) basic_machine=sparc-sun os=-sunos4 ;; sun4sol2) basic_machine=sparc-sun os=-solaris2 ;; sun3 | sun3-*) basic_machine=m68k-sun ;; sun4) basic_machine=sparc-sun ;; sun386 | sun386i | roadrunner) basic_machine=i386-sun ;; sv1) basic_machine=sv1-cray os=-unicos ;; symmetry) basic_machine=i386-sequent os=-dynix ;; t3e) basic_machine=alphaev5-cray os=-unicos ;; t90) basic_machine=t90-cray os=-unicos ;; tile*) basic_machine=$basic_machine-unknown os=-linux-gnu ;; tx39) basic_machine=mipstx39-unknown ;; tx39el) basic_machine=mipstx39el-unknown ;; toad1) basic_machine=pdp10-xkl os=-tops20 ;; tower | tower-32) basic_machine=m68k-ncr ;; tpf) basic_machine=s390x-ibm os=-tpf ;; udi29k) basic_machine=a29k-amd os=-udi ;; ultra3) basic_machine=a29k-nyu os=-sym1 ;; v810 | necv810) basic_machine=v810-nec os=-none ;; vaxv) basic_machine=vax-dec os=-sysv ;; vms) basic_machine=vax-dec os=-vms ;; vpp*|vx|vx-*) basic_machine=f301-fujitsu ;; vxworks960) basic_machine=i960-wrs os=-vxworks ;; vxworks68) basic_machine=m68k-wrs os=-vxworks ;; vxworks29k) basic_machine=a29k-wrs os=-vxworks ;; w65*) basic_machine=w65-wdc os=-none ;; w89k-*) basic_machine=hppa1.1-winbond os=-proelf ;; x64) basic_machine=x86_64-pc ;; xbox) basic_machine=i686-pc os=-mingw32 ;; xps | xps100) basic_machine=xps100-honeywell ;; xscale-* | xscalee[bl]-*) basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'` ;; ymp) basic_machine=ymp-cray os=-unicos ;; none) basic_machine=none-none os=-none ;; # Here we handle the default manufacturer of certain CPU types. It is in # some cases the only manufacturer, in others, it is the most popular. w89k) basic_machine=hppa1.1-winbond ;; op50n) basic_machine=hppa1.1-oki ;; op60c) basic_machine=hppa1.1-oki ;; romp) basic_machine=romp-ibm ;; mmix) basic_machine=mmix-knuth ;; rs6000) basic_machine=rs6000-ibm ;; vax) basic_machine=vax-dec ;; pdp11) basic_machine=pdp11-dec ;; we32k) basic_machine=we32k-att ;; sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) basic_machine=sh-unknown ;; cydra) basic_machine=cydra-cydrome ;; orion) basic_machine=orion-highlevel ;; orion105) basic_machine=clipper-highlevel ;; mac | mpw | mac-mpw) basic_machine=m68k-apple ;; pmac | pmac-mpw) basic_machine=powerpc-apple ;; *-unknown) # Make sure to match an already-canonicalized machine name. ;; *) echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 exit 1 ;; esac # Here we canonicalize certain aliases for manufacturers. case $basic_machine in *-digital*) basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` ;; *-commodore*) basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` ;; *) ;; esac # Decode manufacturer-specific aliases for certain operating systems. if [ x"$os" != x"" ] then case $os in # First match some system type aliases that might get confused # with valid system types. # -solaris* is a basic system type, with this one exception. -auroraux) os=-auroraux ;; -solaris1 | -solaris1.*) os=`echo $os | sed -e 's|solaris1|sunos4|'` ;; -solaris) os=-solaris2 ;; -unixware*) os=-sysv4.2uw ;; -gnu/linux*) os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` ;; # es1800 is here to avoid being matched by es* (a different OS) -es1800*) os=-ose ;; # Now accept the basic system types. # The portable systems comes first. # Each alternative MUST end in a * to match a version number. # -sysv* is not here because it comes later, after sysvr4. -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ | -aos* | -aros* | -cloudabi* | -sortix* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ | -hiux* | -knetbsd* | -mirbsd* | -netbsd* \ | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ | -chorusos* | -chorusrdb* | -cegcc* | -glidix* \ | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ | -linux-newlib* | -linux-musl* | -linux-uclibc* \ | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -windiss* \ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox* | -bme* \ | -midnightbsd*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) case $basic_machine in x86-* | i*86-*) ;; *) os=-nto$os ;; esac ;; -nto-qnx*) ;; -nto*) os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; -sim | -xray | -os68k* | -v88r* \ | -windows* | -osx | -abug | -netware* | -os9* \ | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) ;; -mac*) os=`echo "$os" | sed -e 's|mac|macos|'` ;; -linux-dietlibc) os=-linux-dietlibc ;; -linux*) os=`echo $os | sed -e 's|linux|linux-gnu|'` ;; -sunos5*) os=`echo "$os" | sed -e 's|sunos5|solaris2|'` ;; -sunos6*) os=`echo "$os" | sed -e 's|sunos6|solaris3|'` ;; -opened*) os=-openedition ;; -os400*) os=-os400 ;; -wince*) os=-wince ;; -utek*) os=-bsd ;; -dynix*) os=-bsd ;; -acis*) os=-aos ;; -atheos*) os=-atheos ;; -syllable*) os=-syllable ;; -386bsd) os=-bsd ;; -ctix* | -uts*) os=-sysv ;; -nova*) os=-rtmk-nova ;; -ns2) os=-nextstep2 ;; -nsk*) os=-nsk ;; # Preserve the version number of sinix5. -sinix5.*) os=`echo $os | sed -e 's|sinix|sysv|'` ;; -sinix*) os=-sysv4 ;; -tpf*) os=-tpf ;; -triton*) os=-sysv3 ;; -oss*) os=-sysv3 ;; -svr4*) os=-sysv4 ;; -svr3) os=-sysv3 ;; -sysvr4) os=-sysv4 ;; # This must come after -sysvr4. -sysv*) ;; -ose*) os=-ose ;; -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) os=-mint ;; -zvmoe) os=-zvmoe ;; -dicos*) os=-dicos ;; -pikeos*) # Until real need of OS specific support for # particular features comes up, bare metal # configurations are quite functional. case $basic_machine in arm*) os=-eabi ;; *) os=-elf ;; esac ;; -nacl*) ;; -ios) ;; -none) ;; *) # Get rid of the `-' at the beginning of $os. os=`echo $os | sed 's/[^-]*-//'` echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2 exit 1 ;; esac else # Here we handle the default operating systems that come with various machines. # The value should be what the vendor currently ships out the door with their # machine or put another way, the most popular os provided with the machine. # Note that if you're going to try to match "-MANUFACTURER" here (say, # "-sun"), then you have to tell the case statement up towards the top # that MANUFACTURER isn't an operating system. Otherwise, code above # will signal an error saying that MANUFACTURER isn't an operating # system, and we'll never get to this point. case $basic_machine in score-*) os=-elf ;; spu-*) os=-elf ;; *-acorn) os=-riscix1.2 ;; arm*-rebel) os=-linux ;; arm*-semi) os=-aout ;; c4x-* | tic4x-*) os=-coff ;; c8051-*) os=-elf ;; hexagon-*) os=-elf ;; tic54x-*) os=-coff ;; tic55x-*) os=-coff ;; tic6x-*) os=-coff ;; # This must come before the *-dec entry. pdp10-*) os=-tops20 ;; pdp11-*) os=-none ;; *-dec | vax-*) os=-ultrix4.2 ;; m68*-apollo) os=-domain ;; i386-sun) os=-sunos4.0.2 ;; m68000-sun) os=-sunos3 ;; m68*-cisco) os=-aout ;; mep-*) os=-elf ;; mips*-cisco) os=-elf ;; mips*-*) os=-elf ;; or32-*) os=-coff ;; *-tti) # must be before sparc entry or we get the wrong os. os=-sysv3 ;; sparc-* | *-sun) os=-sunos4.1.1 ;; pru-*) os=-elf ;; *-be) os=-beos ;; *-ibm) os=-aix ;; *-knuth) os=-mmixware ;; *-wec) os=-proelf ;; *-winbond) os=-proelf ;; *-oki) os=-proelf ;; *-hp) os=-hpux ;; *-hitachi) os=-hiux ;; i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) os=-sysv ;; *-cbm) os=-amigaos ;; *-dg) os=-dgux ;; *-dolphin) os=-sysv3 ;; m68k-ccur) os=-rtu ;; m88k-omron*) os=-luna ;; *-next) os=-nextstep ;; *-sequent) os=-ptx ;; *-crds) os=-unos ;; *-ns) os=-genix ;; i370-*) os=-mvs ;; *-gould) os=-sysv ;; *-highlevel) os=-bsd ;; *-encore) os=-bsd ;; *-sgi) os=-irix ;; *-siemens) os=-sysv4 ;; *-masscomp) os=-rtu ;; f30[01]-fujitsu | f700-fujitsu) os=-uxpv ;; *-rom68k) os=-coff ;; *-*bug) os=-coff ;; *-apple) os=-macos ;; *-atari*) os=-mint ;; *) os=-none ;; esac fi # Here we handle the case where we know the os, and the CPU type, but not the # manufacturer. We pick the logical manufacturer. vendor=unknown case $basic_machine in *-unknown) case $os in -riscix*) vendor=acorn ;; -sunos*) vendor=sun ;; -cnk*|-aix*) vendor=ibm ;; -beos*) vendor=be ;; -hpux*) vendor=hp ;; -mpeix*) vendor=hp ;; -hiux*) vendor=hitachi ;; -unos*) vendor=crds ;; -dgux*) vendor=dg ;; -luna*) vendor=omron ;; -genix*) vendor=ns ;; -mvs* | -opened*) vendor=ibm ;; -os400*) vendor=ibm ;; -ptx*) vendor=sequent ;; -tpf*) vendor=ibm ;; -vxsim* | -vxworks* | -windiss*) vendor=wrs ;; -aux*) vendor=apple ;; -hms*) vendor=hitachi ;; -mpw* | -macos*) vendor=apple ;; -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) vendor=atari ;; -vos*) vendor=stratus ;; esac basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` ;; esac echo "$basic_machine$os" exit # Local variables: # eval: (add-hook 'write-file-functions 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" # End: memcached-1.5.22/sizes.c0000664000175000017500000000172613553270304011743 00000000000000#include #include "memcached.h" static void display(const char *name, size_t size) { printf("%s\t%d\n", name, (int)size); } int main(int argc, char **argv) { display("Slab Stats", sizeof(struct slab_stats)); display("Thread stats", sizeof(struct thread_stats) - (200 * sizeof(struct slab_stats))); display("Global stats", sizeof(struct stats)); display("Settings", sizeof(struct settings)); display("Item (no cas)", sizeof(item)); display("Item (cas)", sizeof(item) + sizeof(uint64_t)); #ifdef EXTSTORE display("extstore header", sizeof(item_hdr)); #endif display("Libevent thread", sizeof(LIBEVENT_THREAD) - sizeof(struct thread_stats)); display("Connection", sizeof(conn)); printf("----------------------------------------\n"); display("libevent thread cumulative", sizeof(LIBEVENT_THREAD)); display("Thread stats cumulative\t", sizeof(struct thread_stats)); return 0; } memcached-1.5.22/install-sh0000755000175000017500000003546313256032133012445 00000000000000#!/bin/sh # install - install a program, script, or datafile scriptversion=2014-09-12.12; # UTC # This originates from X11R5 (mit/util/scripts/install.sh), which was # later released in X11R6 (xc/config/util/install.sh) with the # following copyright and license. # # Copyright (C) 1994 X Consortium # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to # deal in the Software without restriction, including without limitation the # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or # sell copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- # TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # Except as contained in this notice, the name of the X Consortium shall not # be used in advertising or otherwise to promote the sale, use or other deal- # ings in this Software without prior written authorization from the X Consor- # tium. # # # FSF changes to this file are in the public domain. # # Calling this script install-sh is preferred over install.sh, to prevent # 'make' implicit rules from creating a file called install from it # when there is no Makefile. # # This script is compatible with the BSD install script, but was written # from scratch. tab=' ' nl=' ' IFS=" $tab$nl" # Set DOITPROG to "echo" to test this script. doit=${DOITPROG-} doit_exec=${doit:-exec} # Put in absolute file names if you don't have them in your path; # or use environment vars. chgrpprog=${CHGRPPROG-chgrp} chmodprog=${CHMODPROG-chmod} chownprog=${CHOWNPROG-chown} cmpprog=${CMPPROG-cmp} cpprog=${CPPROG-cp} mkdirprog=${MKDIRPROG-mkdir} mvprog=${MVPROG-mv} rmprog=${RMPROG-rm} stripprog=${STRIPPROG-strip} posix_mkdir= # Desired mode of installed file. mode=0755 chgrpcmd= chmodcmd=$chmodprog chowncmd= mvcmd=$mvprog rmcmd="$rmprog -f" stripcmd= src= dst= dir_arg= dst_arg= copy_on_change=false is_target_a_directory=possibly usage="\ Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE or: $0 [OPTION]... SRCFILES... DIRECTORY or: $0 [OPTION]... -t DIRECTORY SRCFILES... or: $0 [OPTION]... -d DIRECTORIES... In the 1st form, copy SRCFILE to DSTFILE. In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. In the 4th, create DIRECTORIES. Options: --help display this help and exit. --version display version info and exit. -c (ignored) -C install only if different (preserve the last data modification time) -d create directories instead of installing files. -g GROUP $chgrpprog installed files to GROUP. -m MODE $chmodprog installed files to MODE. -o USER $chownprog installed files to USER. -s $stripprog installed files. -t DIRECTORY install into DIRECTORY. -T report an error if DSTFILE is a directory. Environment variables override the default commands: CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG RMPROG STRIPPROG " while test $# -ne 0; do case $1 in -c) ;; -C) copy_on_change=true;; -d) dir_arg=true;; -g) chgrpcmd="$chgrpprog $2" shift;; --help) echo "$usage"; exit $?;; -m) mode=$2 case $mode in *' '* | *"$tab"* | *"$nl"* | *'*'* | *'?'* | *'['*) echo "$0: invalid mode: $mode" >&2 exit 1;; esac shift;; -o) chowncmd="$chownprog $2" shift;; -s) stripcmd=$stripprog;; -t) is_target_a_directory=always dst_arg=$2 # Protect names problematic for 'test' and other utilities. case $dst_arg in -* | [=\(\)!]) dst_arg=./$dst_arg;; esac shift;; -T) is_target_a_directory=never;; --version) echo "$0 $scriptversion"; exit $?;; --) shift break;; -*) echo "$0: invalid option: $1" >&2 exit 1;; *) break;; esac shift done # We allow the use of options -d and -T together, by making -d # take the precedence; this is for compatibility with GNU install. if test -n "$dir_arg"; then if test -n "$dst_arg"; then echo "$0: target directory not allowed when installing a directory." >&2 exit 1 fi fi if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then # When -d is used, all remaining arguments are directories to create. # When -t is used, the destination is already specified. # Otherwise, the last argument is the destination. Remove it from $@. for arg do if test -n "$dst_arg"; then # $@ is not empty: it contains at least $arg. set fnord "$@" "$dst_arg" shift # fnord fi shift # arg dst_arg=$arg # Protect names problematic for 'test' and other utilities. case $dst_arg in -* | [=\(\)!]) dst_arg=./$dst_arg;; esac done fi if test $# -eq 0; then if test -z "$dir_arg"; then echo "$0: no input file specified." >&2 exit 1 fi # It's OK to call 'install-sh -d' without argument. # This can happen when creating conditional directories. exit 0 fi if test -z "$dir_arg"; then if test $# -gt 1 || test "$is_target_a_directory" = always; then if test ! -d "$dst_arg"; then echo "$0: $dst_arg: Is not a directory." >&2 exit 1 fi fi fi if test -z "$dir_arg"; then do_exit='(exit $ret); exit $ret' trap "ret=129; $do_exit" 1 trap "ret=130; $do_exit" 2 trap "ret=141; $do_exit" 13 trap "ret=143; $do_exit" 15 # Set umask so as not to create temps with too-generous modes. # However, 'strip' requires both read and write access to temps. case $mode in # Optimize common cases. *644) cp_umask=133;; *755) cp_umask=22;; *[0-7]) if test -z "$stripcmd"; then u_plus_rw= else u_plus_rw='% 200' fi cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; *) if test -z "$stripcmd"; then u_plus_rw= else u_plus_rw=,u+rw fi cp_umask=$mode$u_plus_rw;; esac fi for src do # Protect names problematic for 'test' and other utilities. case $src in -* | [=\(\)!]) src=./$src;; esac if test -n "$dir_arg"; then dst=$src dstdir=$dst test -d "$dstdir" dstdir_status=$? else # Waiting for this to be detected by the "$cpprog $src $dsttmp" command # might cause directories to be created, which would be especially bad # if $src (and thus $dsttmp) contains '*'. if test ! -f "$src" && test ! -d "$src"; then echo "$0: $src does not exist." >&2 exit 1 fi if test -z "$dst_arg"; then echo "$0: no destination specified." >&2 exit 1 fi dst=$dst_arg # If destination is a directory, append the input filename; won't work # if double slashes aren't ignored. if test -d "$dst"; then if test "$is_target_a_directory" = never; then echo "$0: $dst_arg: Is a directory" >&2 exit 1 fi dstdir=$dst dst=$dstdir/`basename "$src"` dstdir_status=0 else dstdir=`dirname "$dst"` test -d "$dstdir" dstdir_status=$? fi fi obsolete_mkdir_used=false if test $dstdir_status != 0; then case $posix_mkdir in '') # Create intermediate dirs using mode 755 as modified by the umask. # This is like FreeBSD 'install' as of 1997-10-28. umask=`umask` case $stripcmd.$umask in # Optimize common cases. *[2367][2367]) mkdir_umask=$umask;; .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; *[0-7]) mkdir_umask=`expr $umask + 22 \ - $umask % 100 % 40 + $umask % 20 \ - $umask % 10 % 4 + $umask % 2 `;; *) mkdir_umask=$umask,go-w;; esac # With -d, create the new directory with the user-specified mode. # Otherwise, rely on $mkdir_umask. if test -n "$dir_arg"; then mkdir_mode=-m$mode else mkdir_mode= fi posix_mkdir=false case $umask in *[123567][0-7][0-7]) # POSIX mkdir -p sets u+wx bits regardless of umask, which # is incompatible with FreeBSD 'install' when (umask & 300) != 0. ;; *) # $RANDOM is not portable (e.g. dash); use it when possible to # lower collision chance tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ trap 'ret=$?; rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null; exit $ret' 0 # As "mkdir -p" follows symlinks and we work in /tmp possibly; so # create the $tmpdir first (and fail if unsuccessful) to make sure # that nobody tries to guess the $tmpdir name. if (umask $mkdir_umask && $mkdirprog $mkdir_mode "$tmpdir" && exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1 then if test -z "$dir_arg" || { # Check for POSIX incompatibilities with -m. # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or # other-writable bit of parent directory when it shouldn't. # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. test_tmpdir="$tmpdir/a" ls_ld_tmpdir=`ls -ld "$test_tmpdir"` case $ls_ld_tmpdir in d????-?r-*) different_mode=700;; d????-?--*) different_mode=755;; *) false;; esac && $mkdirprog -m$different_mode -p -- "$test_tmpdir" && { ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"` test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" } } then posix_mkdir=: fi rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" else # Remove any dirs left behind by ancient mkdir implementations. rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null fi trap '' 0;; esac;; esac if $posix_mkdir && ( umask $mkdir_umask && $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" ) then : else # The umask is ridiculous, or mkdir does not conform to POSIX, # or it failed possibly due to a race condition. Create the # directory the slow way, step by step, checking for races as we go. case $dstdir in /*) prefix='/';; [-=\(\)!]*) prefix='./';; *) prefix='';; esac oIFS=$IFS IFS=/ set -f set fnord $dstdir shift set +f IFS=$oIFS prefixes= for d do test X"$d" = X && continue prefix=$prefix$d if test -d "$prefix"; then prefixes= else if $posix_mkdir; then (umask=$mkdir_umask && $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break # Don't fail if two instances are running concurrently. test -d "$prefix" || exit 1 else case $prefix in *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; *) qprefix=$prefix;; esac prefixes="$prefixes '$qprefix'" fi fi prefix=$prefix/ done if test -n "$prefixes"; then # Don't fail if two instances are running concurrently. (umask $mkdir_umask && eval "\$doit_exec \$mkdirprog $prefixes") || test -d "$dstdir" || exit 1 obsolete_mkdir_used=true fi fi fi if test -n "$dir_arg"; then { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 else # Make a couple of temp file names in the proper directory. dsttmp=$dstdir/_inst.$$_ rmtmp=$dstdir/_rm.$$_ # Trap to clean up those temp files at exit. trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 # Copy the file name to the temp name. (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && # and set any options; do chmod last to preserve setuid bits. # # If any of these fail, we abort the whole thing. If we want to # ignore errors from any of these, just make sure not to ignore # errors from the above "$doit $cpprog $src $dsttmp" command. # { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && # If -C, don't bother to copy if it wouldn't change the file. if $copy_on_change && old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && set -f && set X $old && old=:$2:$4:$5:$6 && set X $new && new=:$2:$4:$5:$6 && set +f && test "$old" = "$new" && $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 then rm -f "$dsttmp" else # Rename the file to the real destination. $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || # The rename failed, perhaps because mv can't rename something else # to itself, or perhaps because mv is so ancient that it does not # support -f. { # Now remove or move aside any old file at destination location. # We try this two ways since rm can't unlink itself on some # systems and the destination file might be busy for other # reasons. In this case, the final cleanup might fail but the new # file should still install successfully. { test ! -f "$dst" || $doit $rmcmd -f "$dst" 2>/dev/null || { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } } || { echo "$0: cannot unlink or rename $dst" >&2 (exit 1); exit 1 } } && # Now rename the file to the real destination. $doit $mvcmd "$dsttmp" "$dst" } fi || exit 1 trap '' 0 fi done # Local variables: # eval: (add-hook 'write-file-hooks 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" # time-stamp-time-zone: "UTC" # time-stamp-end: "; # UTC" # End: memcached-1.5.22/storage.h0000664000175000017500000000067213553270304012256 00000000000000#ifndef STORAGE_H #define STORAGE_H int start_storage_write_thread(void *arg); void storage_write_pause(void); void storage_write_resume(void); int start_storage_compact_thread(void *arg); void storage_compact_pause(void); void storage_compact_resume(void); struct extstore_conf_file *storage_conf_parse(char *arg, unsigned int page_size); // Ignore pointers and header bits from the CRC #define STORE_OFFSET offsetof(item, nbytes) #endif memcached-1.5.22/items.h0000664000175000017500000000653313575022500011732 00000000000000#define HOT_LRU 0 #define WARM_LRU 64 #define COLD_LRU 128 #define TEMP_LRU 192 #define CLEAR_LRU(id) (id & ~(3<<6)) #define GET_LRU(id) (id & (3<<6)) /* See items.c */ uint64_t get_cas_id(void); void set_cas_id(uint64_t new_cas); /*@null@*/ item *do_item_alloc(char *key, const size_t nkey, const unsigned int flags, const rel_time_t exptime, const int nbytes); item_chunk *do_item_alloc_chunk(item_chunk *ch, const size_t bytes_remain); item *do_item_alloc_pull(const size_t ntotal, const unsigned int id); void item_free(item *it); bool item_size_ok(const size_t nkey, const int flags, const int nbytes); int do_item_link(item *it, const uint32_t hv); /** may fail if transgresses limits */ void do_item_unlink(item *it, const uint32_t hv); void do_item_unlink_nolock(item *it, const uint32_t hv); void do_item_remove(item *it); void do_item_update(item *it); /** update LRU time to current and reposition */ void do_item_update_nolock(item *it); int do_item_replace(item *it, item *new_it, const uint32_t hv); void do_item_link_fixup(item *it); int item_is_flushed(item *it); unsigned int do_get_lru_size(uint32_t id); void do_item_linktail_q(item *it); void do_item_unlinktail_q(item *it); item *do_item_crawl_q(item *it); void *item_lru_bump_buf_create(void); #define LRU_PULL_EVICT 1 #define LRU_PULL_CRAWL_BLOCKS 2 #define LRU_PULL_RETURN_ITEM 4 /* fill info struct if available */ struct lru_pull_tail_return { item *it; uint32_t hv; }; int lru_pull_tail(const int orig_id, const int cur_lru, const uint64_t total_bytes, const uint8_t flags, const rel_time_t max_age, struct lru_pull_tail_return *ret_it); /*@null@*/ char *item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes); void item_stats(ADD_STAT add_stats, void *c); void do_item_stats_add_crawl(const int i, const uint64_t reclaimed, const uint64_t unfetched, const uint64_t checked); void item_stats_totals(ADD_STAT add_stats, void *c); /*@null@*/ void item_stats_sizes(ADD_STAT add_stats, void *c); void item_stats_sizes_init(void); void item_stats_sizes_enable(ADD_STAT add_stats, void *c); void item_stats_sizes_disable(ADD_STAT add_stats, void *c); void item_stats_sizes_add(item *it); void item_stats_sizes_remove(item *it); bool item_stats_sizes_status(void); /* stats getter for slab automover */ typedef struct { int64_t evicted; int64_t outofmemory; uint32_t age; } item_stats_automove; void fill_item_stats_automove(item_stats_automove *am); item *do_item_get(const char *key, const size_t nkey, const uint32_t hv, conn *c, const bool do_update); item *do_item_touch(const char *key, const size_t nkey, uint32_t exptime, const uint32_t hv, conn *c); void do_item_bump(conn *c, item *it, const uint32_t hv); void item_stats_reset(void); extern pthread_mutex_t lru_locks[POWER_LARGEST]; int start_lru_maintainer_thread(void *arg); int stop_lru_maintainer_thread(void); int init_lru_maintainer(void); void lru_maintainer_pause(void); void lru_maintainer_resume(void); void *lru_bump_buf_create(void); #ifdef EXTSTORE #define STORAGE_delete(e, it) \ do { \ if (it->it_flags & ITEM_HDR) { \ item_hdr *hdr = (item_hdr *)ITEM_data(it); \ extstore_delete(e, hdr->page_id, hdr->page_version, \ 1, ITEM_ntotal(it)); \ } \ } while (0) #else #define STORAGE_delete(...) #endif memcached-1.5.22/testapp.c0000664000175000017500000023035613615376577012313 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #undef NDEBUG #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" #include "cache.h" #include "hash.h" #include "stats_prefix.h" #include "util.h" #include "protocol_binary.h" #ifdef TLS #include #endif #define TMP_TEMPLATE "/tmp/test_file.XXXXXXX" enum test_return { TEST_SKIP, TEST_PASS, TEST_FAIL }; struct conn { int sock; #ifdef TLS SSL_CTX *ssl_ctx; SSL *ssl; #endif ssize_t (*read)(struct conn *c, void *buf, size_t count); ssize_t (*write)(struct conn *c, const void *buf, size_t count); }; struct settings { char *hash_algorithm; }; struct settings settings; static ssize_t tcp_read(struct conn *c, void *buf, size_t count); static ssize_t tcp_write(struct conn *c, const void *buf, size_t count); #ifdef TLS static ssize_t ssl_read(struct conn *c, void *buf, size_t count); static ssize_t ssl_write(struct conn *c, const void *buf, size_t count); #endif ssize_t tcp_read(struct conn *c, void *buf, size_t count) { assert(c != NULL); return read(c->sock, buf, count); } ssize_t tcp_write(struct conn *c, const void *buf, size_t count) { assert(c != NULL); return write(c->sock, buf, count); } #ifdef TLS ssize_t ssl_read(struct conn *c, void *buf, size_t count) { assert(c != NULL); return SSL_read(c->ssl, buf, count); } ssize_t ssl_write(struct conn *c, const void *buf, size_t count) { assert(c != NULL); return SSL_write(c->ssl, buf, count); } #endif static pid_t server_pid; static in_port_t port; static struct conn *con = NULL; static bool allow_closed_read = false; static bool enable_ssl = false; static void close_conn() { if (con == NULL) return; #ifdef TLS if (con->ssl) { SSL_shutdown(con->ssl); SSL_free(con->ssl); } if (con->ssl_ctx) SSL_CTX_free(con->ssl_ctx); #endif if (con->sock > 0) close(con->sock); free(con); con = NULL; } static enum test_return cache_create_test(void) { cache_t *cache = cache_create("test", sizeof(uint32_t), sizeof(char*), NULL, NULL); assert(cache != NULL); cache_destroy(cache); return TEST_PASS; } const uint64_t constructor_pattern = 0xdeadcafebabebeef; static int cache_constructor(void *buffer, void *notused1, int notused2) { uint64_t *ptr = buffer; *ptr = constructor_pattern; return 0; } static enum test_return cache_constructor_test(void) { cache_t *cache = cache_create("test", sizeof(uint64_t), sizeof(uint64_t), cache_constructor, NULL); assert(cache != NULL); uint64_t *ptr = cache_alloc(cache); uint64_t pattern = *ptr; cache_free(cache, ptr); cache_destroy(cache); return (pattern == constructor_pattern) ? TEST_PASS : TEST_FAIL; } static int cache_fail_constructor(void *buffer, void *notused1, int notused2) { return 1; } static enum test_return cache_fail_constructor_test(void) { enum test_return ret = TEST_PASS; cache_t *cache = cache_create("test", sizeof(uint64_t), sizeof(uint64_t), cache_fail_constructor, NULL); assert(cache != NULL); uint64_t *ptr = cache_alloc(cache); if (ptr != NULL) { ret = TEST_FAIL; } cache_destroy(cache); return ret; } static void *destruct_data = 0; static void cache_destructor(void *buffer, void *notused) { destruct_data = buffer; } static enum test_return cache_destructor_test(void) { cache_t *cache = cache_create("test", sizeof(uint32_t), sizeof(char*), NULL, cache_destructor); assert(cache != NULL); char *ptr = cache_alloc(cache); cache_free(cache, ptr); cache_destroy(cache); return (ptr == destruct_data) ? TEST_PASS : TEST_FAIL; } static enum test_return cache_reuse_test(void) { int ii; cache_t *cache = cache_create("test", sizeof(uint32_t), sizeof(char*), NULL, NULL); char *ptr = cache_alloc(cache); cache_free(cache, ptr); for (ii = 0; ii < 100; ++ii) { char *p = cache_alloc(cache); assert(p == ptr); cache_free(cache, ptr); } cache_destroy(cache); return TEST_PASS; } static enum test_return cache_bulkalloc(size_t datasize) { cache_t *cache = cache_create("test", datasize, sizeof(char*), NULL, NULL); #define ITERATIONS 1024 void *ptr[ITERATIONS]; for (int ii = 0; ii < ITERATIONS; ++ii) { ptr[ii] = cache_alloc(cache); assert(ptr[ii] != 0); memset(ptr[ii], 0xff, datasize); } for (int ii = 0; ii < ITERATIONS; ++ii) { cache_free(cache, ptr[ii]); } #undef ITERATIONS cache_destroy(cache); return TEST_PASS; } static enum test_return test_issue_161(void) { enum test_return ret = cache_bulkalloc(1); if (ret == TEST_PASS) { ret = cache_bulkalloc(512); } return ret; } static enum test_return cache_redzone_test(void) { #ifndef HAVE_UMEM_H cache_t *cache = cache_create("test", sizeof(uint32_t), sizeof(char*), NULL, NULL); /* Ignore SIGABRT */ struct sigaction old_action; struct sigaction action = { .sa_handler = SIG_IGN, .sa_flags = 0}; sigemptyset(&action.sa_mask); sigaction(SIGABRT, &action, &old_action); /* check memory debug.. */ char *p = cache_alloc(cache); char old = *(p - 1); *(p - 1) = 0; cache_free(cache, p); assert(cache_error == -1); *(p - 1) = old; p[sizeof(uint32_t)] = 0; cache_free(cache, p); assert(cache_error == 1); /* restore signal handler */ sigaction(SIGABRT, &old_action, NULL); cache_destroy(cache); return TEST_PASS; #else return TEST_SKIP; #endif } static enum test_return test_stats_prefix_find(void) { PREFIX_STATS *pfs1, *pfs2; stats_prefix_clear(); pfs1 = stats_prefix_find("abc", 3); assert(pfs1 == NULL); pfs1 = stats_prefix_find("abc|", 4); assert(pfs1 == NULL); pfs1 = stats_prefix_find("abc:", 4); assert(pfs1 != NULL); assert(0ULL == (pfs1->num_gets + pfs1->num_sets + pfs1->num_deletes + pfs1->num_hits)); pfs2 = stats_prefix_find("abc:", 4); assert(pfs1 == pfs2); pfs2 = stats_prefix_find("abc:d", 5); assert(pfs1 == pfs2); pfs2 = stats_prefix_find("xyz123:", 6); assert(pfs1 != pfs2); pfs2 = stats_prefix_find("ab:", 3); assert(pfs1 != pfs2); return TEST_PASS; } static enum test_return test_stats_prefix_record_get(void) { PREFIX_STATS *pfs; stats_prefix_clear(); stats_prefix_record_get("abc:123", 7, false); pfs = stats_prefix_find("abc:123", 7); assert(1 == pfs->num_gets); assert(0 == pfs->num_hits); stats_prefix_record_get("abc:456", 7, false); assert(2 == pfs->num_gets); assert(0 == pfs->num_hits); stats_prefix_record_get("abc:456", 7, true); assert(3 == pfs->num_gets); assert(1 == pfs->num_hits); stats_prefix_record_get("def:", 4, true); assert(3 == pfs->num_gets); assert(1 == pfs->num_hits); return TEST_PASS; } static enum test_return test_stats_prefix_record_delete(void) { PREFIX_STATS *pfs; stats_prefix_clear(); stats_prefix_record_delete("abc:123", 7); pfs = stats_prefix_find("abc:123", 7); assert(0 == pfs->num_gets); assert(0 == pfs->num_hits); assert(1 == pfs->num_deletes); assert(0 == pfs->num_sets); stats_prefix_record_delete("def:", 4); assert(1 == pfs->num_deletes); return TEST_PASS; } static enum test_return test_stats_prefix_record_set(void) { PREFIX_STATS *pfs; stats_prefix_clear(); stats_prefix_record_set("abc:123", 7); pfs = stats_prefix_find("abc:123", 7); assert(0 == pfs->num_gets); assert(0 == pfs->num_hits); assert(0 == pfs->num_deletes); assert(1 == pfs->num_sets); stats_prefix_record_delete("def:", 4); assert(1 == pfs->num_sets); return TEST_PASS; } static enum test_return test_stats_prefix_dump(void) { int hashval = hash("abc", 3) % PREFIX_HASH_SIZE; char tmp[500]; char *expected; int keynum; int length; stats_prefix_clear(); assert(strcmp("END\r\n", stats_prefix_dump(&length)) == 0); assert(5 == length); stats_prefix_record_set("abc:123", 7); expected = "PREFIX abc get 0 hit 0 set 1 del 0\r\nEND\r\n"; assert(strcmp(expected, stats_prefix_dump(&length)) == 0); assert(strlen(expected) == length); stats_prefix_record_get("abc:123", 7, false); expected = "PREFIX abc get 1 hit 0 set 1 del 0\r\nEND\r\n"; assert(strcmp(expected, stats_prefix_dump(&length)) == 0); assert(strlen(expected) == length); stats_prefix_record_get("abc:123", 7, true); expected = "PREFIX abc get 2 hit 1 set 1 del 0\r\nEND\r\n"; assert(strcmp(expected, stats_prefix_dump(&length)) == 0); assert(strlen(expected) == length); stats_prefix_record_delete("abc:123", 7); expected = "PREFIX abc get 2 hit 1 set 1 del 1\r\nEND\r\n"; assert(strcmp(expected, stats_prefix_dump(&length)) == 0); assert(strlen(expected) == length); /* The order of results might change if we switch hash functions. */ stats_prefix_record_delete("def:123", 7); expected = "PREFIX abc get 2 hit 1 set 1 del 1\r\n" "PREFIX def get 0 hit 0 set 0 del 1\r\n" "END\r\n"; assert(strcmp(expected, stats_prefix_dump(&length)) == 0); assert(strlen(expected) == length); /* Find a key that hashes to the same bucket as "abc" */ bool found_match = false; for (keynum = 0; keynum < PREFIX_HASH_SIZE * 100; keynum++) { snprintf(tmp, sizeof(tmp), "%d:", keynum); /* -1 because only the prefix portion is used when hashing */ if (hashval == hash(tmp, strlen(tmp) - 1) % PREFIX_HASH_SIZE) { found_match = true; break; } } assert(found_match); stats_prefix_record_set(tmp, strlen(tmp)); snprintf(tmp, sizeof(tmp), "PREFIX %d get 0 hit 0 set 1 del 0\r\n" "PREFIX abc get 2 hit 1 set 1 del 1\r\n" "PREFIX def get 0 hit 0 set 0 del 1\r\n" "END\r\n", keynum); assert(strcmp(tmp, stats_prefix_dump(&length)) == 0); assert(strlen(tmp) == length); return TEST_PASS; } static enum test_return test_safe_strtoul(void) { uint32_t val; assert(safe_strtoul("123", &val)); assert(val == 123); assert(safe_strtoul("+123", &val)); assert(val == 123); assert(!safe_strtoul("", &val)); // empty assert(!safe_strtoul("123BOGUS", &val)); // non-numeric assert(!safe_strtoul(" issue221", &val)); // non-numeric /* Not sure what it does, but this works with ICC :/ assert(!safe_strtoul("92837498237498237498029383", &val)); // out of range */ // extremes: assert(safe_strtoul("4294967295", &val)); // 2**32 - 1 assert(val == 4294967295L); /* This actually works on 64-bit ubuntu assert(!safe_strtoul("4294967296", &val)); // 2**32 */ assert(!safe_strtoul("-1", &val)); // negative return TEST_PASS; } static enum test_return test_safe_strtoull(void) { uint64_t val; assert(safe_strtoull("123", &val)); assert(val == 123); assert(safe_strtoull("+123", &val)); assert(val == 123); assert(!safe_strtoull("", &val)); // empty assert(!safe_strtoull("123BOGUS", &val)); // non-numeric assert(!safe_strtoull("92837498237498237498029383", &val)); // out of range assert(!safe_strtoull(" issue221", &val)); // non-numeric // extremes: assert(safe_strtoull("18446744073709551615", &val)); // 2**64 - 1 assert(val == 18446744073709551615ULL); assert(!safe_strtoull("18446744073709551616", &val)); // 2**64 assert(!safe_strtoull("-1", &val)); // negative return TEST_PASS; } static enum test_return test_safe_strtoll(void) { int64_t val; assert(safe_strtoll("123", &val)); assert(val == 123); assert(safe_strtoll("+123", &val)); assert(val == 123); assert(safe_strtoll("-123", &val)); assert(val == -123); assert(!safe_strtoll("", &val)); // empty assert(!safe_strtoll("123BOGUS", &val)); // non-numeric assert(!safe_strtoll("92837498237498237498029383", &val)); // out of range assert(!safe_strtoll(" issue221", &val)); // non-numeric // extremes: assert(!safe_strtoll("18446744073709551615", &val)); // 2**64 - 1 assert(safe_strtoll("9223372036854775807", &val)); // 2**63 - 1 assert(val == 9223372036854775807LL); /* assert(safe_strtoll("-9223372036854775808", &val)); // -2**63 assert(val == -9223372036854775808LL); */ assert(!safe_strtoll("-9223372036854775809", &val)); // -2**63 - 1 // We'll allow space to terminate the string. And leading space. assert(safe_strtoll(" 123 foo", &val)); assert(val == 123); return TEST_PASS; } static enum test_return test_safe_strtol(void) { int32_t val; assert(safe_strtol("123", &val)); assert(val == 123); assert(safe_strtol("+123", &val)); assert(val == 123); assert(safe_strtol("-123", &val)); assert(val == -123); assert(!safe_strtol("", &val)); // empty assert(!safe_strtol("123BOGUS", &val)); // non-numeric assert(!safe_strtol("92837498237498237498029383", &val)); // out of range assert(!safe_strtol(" issue221", &val)); // non-numeric // extremes: /* This actually works on 64-bit ubuntu assert(!safe_strtol("2147483648", &val)); // (expt 2.0 31.0) */ assert(safe_strtol("2147483647", &val)); // (- (expt 2.0 31) 1) assert(val == 2147483647L); /* This actually works on 64-bit ubuntu assert(!safe_strtol("-2147483649", &val)); // (- (expt -2.0 31) 1) */ // We'll allow space to terminate the string. And leading space. assert(safe_strtol(" 123 foo", &val)); assert(val == 123); return TEST_PASS; } /** * Function to start the server and let it listen on a random port * * @param port_out where to store the TCP port number the server is * listening on * @param daemon set to true if you want to run the memcached server * as a daemon process * @return the pid of the memcached server */ static pid_t start_server(in_port_t *port_out, bool daemon, int timeout) { char environment[80]; snprintf(environment, sizeof(environment), "MEMCACHED_PORT_FILENAME=/tmp/ports.%lu", (long)getpid()); char *filename= environment + strlen("MEMCACHED_PORT_FILENAME="); char pid_file[80]; snprintf(pid_file, sizeof(pid_file), "/tmp/pid.%lu", (long)getpid()); remove(filename); remove(pid_file); #ifdef __sun /* I want to name the corefiles differently so that they don't overwrite each other */ char coreadm[128]; snprintf(coreadm, sizeof(coreadm), "coreadm -p core.%%f.%%p %lu", (unsigned long)getpid()); system(coreadm); #endif pid_t pid = fork(); assert(pid != -1); if (pid == 0) { /* Child */ char *argv[24]; int arg = 0; char tmo[24]; snprintf(tmo, sizeof(tmo), "%u", timeout); putenv(environment); #ifdef __sun putenv("LD_PRELOAD=watchmalloc.so.1"); putenv("MALLOC_DEBUG=WATCH"); #endif if (!daemon) { argv[arg++] = "./timedrun"; argv[arg++] = tmo; } argv[arg++] = "./memcached-debug"; argv[arg++] = "-A"; argv[arg++] = "-p"; argv[arg++] = "-1"; argv[arg++] = "-U"; argv[arg++] = "0"; #ifdef TLS if (enable_ssl) { argv[arg++] = "-Z"; argv[arg++] = "-o"; argv[arg++] = "ssl_chain_cert=t/server_crt.pem"; argv[arg++] = "-o"; argv[arg++] = "ssl_key=t/server_key.pem"; } #endif /* Handle rpmbuild and the like doing this as root */ if (getuid() == 0) { argv[arg++] = "-u"; argv[arg++] = "root"; } if (daemon) { argv[arg++] = "-d"; argv[arg++] = "-P"; argv[arg++] = pid_file; } #ifdef MESSAGE_DEBUG argv[arg++] = "-vvv"; #endif #ifdef HAVE_DROP_PRIVILEGES argv[arg++] = "-o"; argv[arg++] = "relaxed_privileges"; #endif argv[arg++] = NULL; assert(execv(argv[0], argv) != -1); } /* Yeah just let us "busy-wait" for the file to be created ;-) */ while (access(filename, F_OK) == -1) { usleep(10); } FILE *fp = fopen(filename, "r"); if (fp == NULL) { fprintf(stderr, "Failed to open the file containing port numbers: %s\n", strerror(errno)); assert(false); } *port_out = (in_port_t)-1; char buffer[80]; while ((fgets(buffer, sizeof(buffer), fp)) != NULL) { if (strncmp(buffer, "TCP INET: ", 10) == 0) { int32_t val; assert(safe_strtol(buffer + 10, &val)); *port_out = (in_port_t)val; } } fclose(fp); assert(remove(filename) == 0); if (daemon) { /* loop and wait for the pid file.. There is a potential race * condition that the server just created the file but isn't * finished writing the content, so we loop a few times * reading as well */ while (access(pid_file, F_OK) == -1) { usleep(10); } fp = fopen(pid_file, "r"); if (fp == NULL) { fprintf(stderr, "Failed to open pid file: %s\n", strerror(errno)); assert(false); } /* Avoid race by retrying 20 times */ for (int x = 0; x < 20 && fgets(buffer, sizeof(buffer), fp) == NULL; x++) { usleep(10); } fclose(fp); int32_t val; assert(safe_strtol(buffer, &val)); pid = (pid_t)val; } return pid; } static enum test_return test_issue_44(void) { in_port_t port; pid_t pid = start_server(&port, true, 15); assert(kill(pid, SIGHUP) == 0); sleep(1); assert(kill(pid, SIGTERM) == 0); return TEST_PASS; } static struct addrinfo *lookuphost(const char *hostname, in_port_t port) { struct addrinfo *ai = 0; struct addrinfo hints = { .ai_family = AF_UNSPEC, .ai_protocol = IPPROTO_TCP, .ai_socktype = SOCK_STREAM }; char service[NI_MAXSERV]; int error; (void)snprintf(service, NI_MAXSERV, "%d", port); if ((error = getaddrinfo(hostname, service, &hints, &ai)) != 0) { if (error != EAI_SYSTEM) { fprintf(stderr, "getaddrinfo(): %s\n", gai_strerror(error)); } else { perror("getaddrinfo()"); } } return ai; } static struct conn *connect_server(const char *hostname, in_port_t port, bool nonblock, const bool ssl) { struct conn *c; if (!(c = (struct conn *)calloc(1, sizeof(struct conn)))) { fprintf(stderr, "Failed to allocate the client connection: %s\n", strerror(errno)); return NULL; } struct addrinfo *ai = lookuphost(hostname, port); int sock = -1; if (ai != NULL) { if ((sock = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol)) != -1) { if (connect(sock, ai->ai_addr, ai->ai_addrlen) == -1) { fprintf(stderr, "Failed to connect socket: %s\n", strerror(errno)); close(sock); sock = -1; } else if (nonblock) { int flags = fcntl(sock, F_GETFL, 0); if (flags < 0 || fcntl(sock, F_SETFL, flags | O_NONBLOCK) < 0) { fprintf(stderr, "Failed to enable nonblocking mode: %s\n", strerror(errno)); close(sock); sock = -1; } } } else { fprintf(stderr, "Failed to create socket: %s\n", strerror(errno)); } freeaddrinfo(ai); } c->sock = sock; #ifdef TLS if (sock > 0 && ssl) { c->ssl_ctx = SSL_CTX_new(SSLv23_client_method()); if (c->ssl_ctx == NULL) { fprintf(stderr, "Failed to create the SSL context: %s\n", strerror(errno)); close(sock); sock = -1; } c->ssl = SSL_new(c->ssl_ctx); if (c->ssl == NULL) { fprintf(stderr, "Failed to create the SSL object: %s\n", strerror(errno)); close(sock); sock = -1; } SSL_set_fd (c->ssl, c->sock); int ret = SSL_connect(c->ssl); if (ret < 0) { int err = SSL_get_error(c->ssl, ret); if (err == SSL_ERROR_SYSCALL || err == SSL_ERROR_SSL) { fprintf(stderr, "SSL connection failed with error code : %s\n", strerror(errno)); close(sock); sock = -1; } } c->read = ssl_read; c->write = ssl_write; } else #endif { c->read = tcp_read; c->write = tcp_write; } return c; } static enum test_return test_vperror(void) { int rv = 0; int oldstderr = dup(STDERR_FILENO); assert(oldstderr >= 0); char tmpl[sizeof(TMP_TEMPLATE)+1]; strncpy(tmpl, TMP_TEMPLATE, sizeof(TMP_TEMPLATE)+1); int newfile = mkstemp(tmpl); assert(newfile > 0); rv = dup2(newfile, STDERR_FILENO); assert(rv == STDERR_FILENO); rv = close(newfile); assert(rv == 0); errno = EIO; vperror("Old McDonald had a farm. %s", "EI EIO"); /* Restore stderr */ rv = dup2(oldstderr, STDERR_FILENO); assert(rv == STDERR_FILENO); /* Go read the file */ char buf[80] = { 0 }; FILE *efile = fopen(tmpl, "r"); assert(efile); char *prv = fgets(buf, sizeof(buf), efile); assert(prv); fclose(efile); unlink(tmpl); char expected[80] = { 0 }; snprintf(expected, sizeof(expected), "Old McDonald had a farm. EI EIO: %s\n", strerror(EIO)); /* fprintf(stderr, "\nExpected: ``%s''" "\nGot: ``%s''\n", expected, buf); */ return strcmp(expected, buf) == 0 ? TEST_PASS : TEST_FAIL; } static void send_ascii_command(const char *buf) { off_t offset = 0; const char* ptr = buf; size_t len = strlen(buf); do { ssize_t nw = con->write((void*)con, ptr + offset, len - offset); if (nw == -1) { if (errno != EINTR) { fprintf(stderr, "Failed to write: %s\n", strerror(errno)); abort(); } } else { offset += nw; } } while (offset < len); } /* * This is a dead slow single byte read, but it should only read out * _one_ response and I don't have an input buffer... The current * implementation only supports single-line responses, so if you want to use * it for get commands you need to implement that first ;-) */ static void read_ascii_response(char *buffer, size_t size) { off_t offset = 0; bool need_more = true; do { ssize_t nr = con->read(con, buffer + offset, 1); if (nr == -1) { if (errno != EINTR) { fprintf(stderr, "Failed to read: %s\n", strerror(errno)); abort(); } } else { assert(nr == 1); if (buffer[offset] == '\n') { need_more = false; buffer[offset + 1] = '\0'; } offset += nr; assert(offset + 1 < size); } } while (need_more); } static enum test_return test_issue_92(void) { char buffer[1024]; close_conn(); con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); send_ascii_command("stats cachedump 1 0 0\r\n"); read_ascii_response(buffer, sizeof(buffer)); assert(strncmp(buffer, "END", strlen("END")) == 0); send_ascii_command("stats cachedump 200 0 0\r\n"); read_ascii_response(buffer, sizeof(buffer)); assert(strncmp(buffer, "CLIENT_ERROR", strlen("CLIENT_ERROR")) == 0); close_conn(); con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); return TEST_PASS; } static enum test_return test_issue_102(void) { char buffer[4096]; memset(buffer, ' ', sizeof(buffer)); buffer[sizeof(buffer) - 1] = '\0'; close_conn(); con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); send_ascii_command(buffer); /* verify that the server closed the connection */ assert(con->read(con, buffer, sizeof(buffer)) == 0); close_conn(); con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); snprintf(buffer, sizeof(buffer), "gets "); size_t offset = 5; while (offset < 4000) { offset += snprintf(buffer + offset, sizeof(buffer) - offset, "%010u ", (unsigned int)offset); } send_ascii_command(buffer); usleep(250); send_ascii_command("\r\n"); char rsp[80]; read_ascii_response(rsp, sizeof(rsp)); assert(strncmp(rsp, "END", strlen("END")) == 0); buffer[3]= ' '; send_ascii_command(buffer); usleep(250); send_ascii_command("\r\n"); read_ascii_response(rsp, sizeof(rsp)); assert(strncmp(rsp, "END", strlen("END")) == 0); memset(buffer, ' ', sizeof(buffer)); int len = snprintf(buffer + 101, sizeof(buffer) - 101, "gets foo"); buffer[101 + len] = ' '; buffer[sizeof(buffer) - 1] = '\0'; send_ascii_command(buffer); /* verify that the server closed the connection */ assert(con->read(con, buffer, sizeof(buffer)) == 0); close_conn(); con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); return TEST_PASS; } static enum test_return start_memcached_server(void) { server_pid = start_server(&port, false, 600); close_conn(); con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); return TEST_PASS; } static enum test_return stop_memcached_server(void) { close_conn(); if (server_pid != -1) { assert(kill(server_pid, SIGTERM) == 0); } return TEST_PASS; } static enum test_return shutdown_memcached_server(void) { char buffer[1024]; close_conn(); con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); send_ascii_command("shutdown\r\n"); /* verify that the server closed the connection */ assert(con->read(con, buffer, sizeof(buffer)) == 0); close_conn(); /* We set server_pid to -1 so that we don't later call kill() */ if (kill(server_pid, 0) == 0) { server_pid = -1; } return TEST_PASS; } static void safe_send(const void* buf, size_t len, bool hickup) { off_t offset = 0; const char* ptr = buf; #ifdef MESSAGE_DEBUG uint8_t val = *ptr; assert(val == (uint8_t)0x80); fprintf(stderr, "About to send %lu bytes:", (unsigned long)len); for (int ii = 0; ii < len; ++ii) { if (ii % 4 == 0) { fprintf(stderr, "\n "); } val = *(ptr + ii); fprintf(stderr, " 0x%02x", val); } fprintf(stderr, "\n"); usleep(500); #endif do { size_t num_bytes = len - offset; if (hickup) { if (num_bytes > 1024) { num_bytes = (rand() % 1023) + 1; } } ssize_t nw = con->write(con, ptr + offset, num_bytes); if (nw == -1) { if (errno != EINTR) { fprintf(stderr, "Failed to write: %s\n", strerror(errno)); abort(); } } else { if (hickup) { usleep(100); } offset += nw; } } while (offset < len); } static bool safe_recv(void *buf, size_t len) { if (len == 0) { return true; } off_t offset = 0; do { ssize_t nr = con->read(con, ((char*)buf) + offset, len - offset); if (nr == -1) { if (errno != EINTR) { fprintf(stderr, "Failed to read: %s\n", strerror(errno)); abort(); } } else { if (nr == 0 && allow_closed_read) { return false; } assert(nr != 0); offset += nr; } } while (offset < len); return true; } static bool safe_recv_packet(void *buf, size_t size) { protocol_binary_response_no_extras *response = buf; assert(size > sizeof(*response)); if (!safe_recv(response, sizeof(*response))) { return false; } response->message.header.response.keylen = ntohs(response->message.header.response.keylen); response->message.header.response.status = ntohs(response->message.header.response.status); response->message.header.response.bodylen = ntohl(response->message.header.response.bodylen); size_t len = sizeof(*response); char *ptr = buf; ptr += len; if (!safe_recv(ptr, response->message.header.response.bodylen)) { return false; } #ifdef MESSAGE_DEBUG usleep(500); ptr = buf; len += response->message.header.response.bodylen; uint8_t val = *ptr; assert(val == (uint8_t)0x81); fprintf(stderr, "Received %lu bytes:", (unsigned long)len); for (int ii = 0; ii < len; ++ii) { if (ii % 4 == 0) { fprintf(stderr, "\n "); } val = *(ptr + ii); fprintf(stderr, " 0x%02x", val); } fprintf(stderr, "\n"); #endif return true; } static off_t storage_command(char*buf, size_t bufsz, uint8_t cmd, const void* key, size_t keylen, const void* dta, size_t dtalen, uint32_t flags, uint32_t exp) { /* all of the storage commands use the same command layout */ protocol_binary_request_set *request = (void*)buf; assert(bufsz > sizeof(*request) + keylen + dtalen); memset(request, 0, sizeof(*request)); request->message.header.request.magic = PROTOCOL_BINARY_REQ; request->message.header.request.opcode = cmd; request->message.header.request.keylen = htons(keylen); request->message.header.request.extlen = 8; request->message.header.request.bodylen = htonl(keylen + 8 + dtalen); request->message.header.request.opaque = 0xdeadbeef; request->message.body.flags = flags; request->message.body.expiration = exp; off_t key_offset = sizeof(protocol_binary_request_no_extras) + 8; memcpy(buf + key_offset, key, keylen); if (dta != NULL) { memcpy(buf + key_offset + keylen, dta, dtalen); } return key_offset + keylen + dtalen; } static off_t ext_command(char* buf, size_t bufsz, uint8_t cmd, const void* ext, size_t extlen, const void* key, size_t keylen, const void* dta, size_t dtalen) { protocol_binary_request_no_extras *request = (void*)buf; assert(bufsz > sizeof(*request) + extlen + keylen + dtalen); memset(request, 0, sizeof(*request)); request->message.header.request.magic = PROTOCOL_BINARY_REQ; request->message.header.request.opcode = cmd; request->message.header.request.extlen = extlen; request->message.header.request.keylen = htons(keylen); request->message.header.request.bodylen = htonl(extlen + keylen + dtalen); request->message.header.request.opaque = 0xdeadbeef; off_t ext_offset = sizeof(protocol_binary_request_no_extras); off_t key_offset = ext_offset + extlen; off_t dta_offset = key_offset + keylen; if (ext != NULL) { memcpy(buf + ext_offset, ext, extlen); } if (key != NULL) { memcpy(buf + key_offset, key, keylen); } if (dta != NULL) { memcpy(buf + dta_offset, dta, dtalen); } return sizeof(*request) + extlen + keylen + dtalen; } static off_t raw_command(char* buf, size_t bufsz, uint8_t cmd, const void* key, size_t keylen, const void* dta, size_t dtalen) { /* all of the storage commands use the same command layout */ return ext_command(buf, bufsz, cmd, NULL, 0, key, keylen, dta, dtalen); } static off_t flush_command(char* buf, size_t bufsz, uint8_t cmd, uint32_t exptime, bool use_extra) { protocol_binary_request_flush *request = (void*)buf; assert(bufsz > sizeof(*request)); memset(request, 0, sizeof(*request)); request->message.header.request.magic = PROTOCOL_BINARY_REQ; request->message.header.request.opcode = cmd; off_t size = sizeof(protocol_binary_request_no_extras); if (use_extra) { request->message.header.request.extlen = 4; request->message.body.expiration = htonl(exptime); request->message.header.request.bodylen = htonl(4); size += 4; } request->message.header.request.opaque = 0xdeadbeef; return size; } static off_t touch_command(char* buf, size_t bufsz, uint8_t cmd, const void* key, size_t keylen, uint32_t exptime) { protocol_binary_request_touch *request = (void*)buf; assert(bufsz > sizeof(*request)); memset(request, 0, sizeof(*request)); request->message.header.request.magic = PROTOCOL_BINARY_REQ; request->message.header.request.opcode = cmd; request->message.header.request.keylen = htons(keylen); request->message.header.request.extlen = 4; request->message.body.expiration = htonl(exptime); request->message.header.request.bodylen = htonl(keylen + 4); request->message.header.request.opaque = 0xdeadbeef; off_t key_offset = sizeof(protocol_binary_request_no_extras) + 4; memcpy(buf + key_offset, key, keylen); return sizeof(protocol_binary_request_no_extras) + 4 + keylen; } static off_t arithmetic_command(char* buf, size_t bufsz, uint8_t cmd, const void* key, size_t keylen, uint64_t delta, uint64_t initial, uint32_t exp) { protocol_binary_request_incr *request = (void*)buf; assert(bufsz > sizeof(*request) + keylen); memset(request, 0, sizeof(*request)); request->message.header.request.magic = PROTOCOL_BINARY_REQ; request->message.header.request.opcode = cmd; request->message.header.request.keylen = htons(keylen); request->message.header.request.extlen = 20; request->message.header.request.bodylen = htonl(keylen + 20); request->message.header.request.opaque = 0xdeadbeef; request->message.body.delta = htonll(delta); request->message.body.initial = htonll(initial); request->message.body.expiration = htonl(exp); off_t key_offset = sizeof(protocol_binary_request_no_extras) + 20; memcpy(buf + key_offset, key, keylen); return key_offset + keylen; } static void validate_response_header(protocol_binary_response_no_extras *response, uint8_t cmd, uint16_t status) { assert(response->message.header.response.magic == PROTOCOL_BINARY_RES); assert(response->message.header.response.opcode == cmd); assert(response->message.header.response.datatype == PROTOCOL_BINARY_RAW_BYTES); assert(response->message.header.response.status == status); assert(response->message.header.response.opaque == 0xdeadbeef); if (status == PROTOCOL_BINARY_RESPONSE_SUCCESS) { switch (cmd) { case PROTOCOL_BINARY_CMD_ADDQ: case PROTOCOL_BINARY_CMD_APPENDQ: case PROTOCOL_BINARY_CMD_DECREMENTQ: case PROTOCOL_BINARY_CMD_DELETEQ: case PROTOCOL_BINARY_CMD_FLUSHQ: case PROTOCOL_BINARY_CMD_INCREMENTQ: case PROTOCOL_BINARY_CMD_PREPENDQ: case PROTOCOL_BINARY_CMD_QUITQ: case PROTOCOL_BINARY_CMD_REPLACEQ: case PROTOCOL_BINARY_CMD_SETQ: assert("Quiet command shouldn't return on success" == NULL); default: break; } switch (cmd) { case PROTOCOL_BINARY_CMD_ADD: case PROTOCOL_BINARY_CMD_REPLACE: case PROTOCOL_BINARY_CMD_SET: case PROTOCOL_BINARY_CMD_APPEND: case PROTOCOL_BINARY_CMD_PREPEND: assert(response->message.header.response.keylen == 0); assert(response->message.header.response.extlen == 0); assert(response->message.header.response.bodylen == 0); assert(response->message.header.response.cas != 0); break; case PROTOCOL_BINARY_CMD_FLUSH: case PROTOCOL_BINARY_CMD_NOOP: case PROTOCOL_BINARY_CMD_QUIT: case PROTOCOL_BINARY_CMD_DELETE: assert(response->message.header.response.keylen == 0); assert(response->message.header.response.extlen == 0); assert(response->message.header.response.bodylen == 0); assert(response->message.header.response.cas == 0); break; case PROTOCOL_BINARY_CMD_DECREMENT: case PROTOCOL_BINARY_CMD_INCREMENT: assert(response->message.header.response.keylen == 0); assert(response->message.header.response.extlen == 0); assert(response->message.header.response.bodylen == 8); assert(response->message.header.response.cas != 0); break; case PROTOCOL_BINARY_CMD_STAT: assert(response->message.header.response.extlen == 0); /* key and value exists in all packets except in the terminating */ assert(response->message.header.response.cas == 0); break; case PROTOCOL_BINARY_CMD_VERSION: assert(response->message.header.response.keylen == 0); assert(response->message.header.response.extlen == 0); assert(response->message.header.response.bodylen != 0); assert(response->message.header.response.cas == 0); break; case PROTOCOL_BINARY_CMD_GET: case PROTOCOL_BINARY_CMD_GETQ: case PROTOCOL_BINARY_CMD_GAT: case PROTOCOL_BINARY_CMD_GATQ: assert(response->message.header.response.keylen == 0); assert(response->message.header.response.extlen == 4); assert(response->message.header.response.cas != 0); break; case PROTOCOL_BINARY_CMD_GETK: case PROTOCOL_BINARY_CMD_GETKQ: case PROTOCOL_BINARY_CMD_GATK: case PROTOCOL_BINARY_CMD_GATKQ: assert(response->message.header.response.keylen != 0); assert(response->message.header.response.extlen == 4); assert(response->message.header.response.cas != 0); break; default: /* Undefined command code */ break; } } else { assert(response->message.header.response.cas == 0); assert(response->message.header.response.extlen == 0); if (cmd != PROTOCOL_BINARY_CMD_GETK && cmd != PROTOCOL_BINARY_CMD_GATK) { assert(response->message.header.response.keylen == 0); } } } static enum test_return test_binary_noop(void) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } buffer; size_t len = raw_command(buffer.bytes, sizeof(buffer.bytes), PROTOCOL_BINARY_CMD_NOOP, NULL, 0, NULL, 0); safe_send(buffer.bytes, len, false); safe_recv_packet(buffer.bytes, sizeof(buffer.bytes)); validate_response_header(&buffer.response, PROTOCOL_BINARY_CMD_NOOP, PROTOCOL_BINARY_RESPONSE_SUCCESS); return TEST_PASS; } static enum test_return test_binary_quit_impl(uint8_t cmd) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } buffer; size_t len = raw_command(buffer.bytes, sizeof(buffer.bytes), cmd, NULL, 0, NULL, 0); safe_send(buffer.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_QUIT) { safe_recv_packet(buffer.bytes, sizeof(buffer.bytes)); validate_response_header(&buffer.response, PROTOCOL_BINARY_CMD_QUIT, PROTOCOL_BINARY_RESPONSE_SUCCESS); } /* Socket should be closed now, read should return 0 */ assert(con->read(con, buffer.bytes, sizeof(buffer.bytes)) == 0); close_conn(); con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); return TEST_PASS; } static enum test_return test_binary_quit(void) { return test_binary_quit_impl(PROTOCOL_BINARY_CMD_QUIT); } static enum test_return test_binary_quitq(void) { return test_binary_quit_impl(PROTOCOL_BINARY_CMD_QUITQ); } static enum test_return test_binary_set_impl(const char *key, uint8_t cmd) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } send, receive; uint64_t value = 0xdeadbeefdeadcafe; size_t len = storage_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), &value, sizeof(value), 0, 0); /* Set should work over and over again */ int ii; for (ii = 0; ii < 10; ++ii) { safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_SET) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); } } if (cmd == PROTOCOL_BINARY_CMD_SETQ) { return test_binary_noop(); } send.request.message.header.request.cas = receive.response.message.header.response.cas; safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_SET) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); assert(receive.response.message.header.response.cas != send.request.message.header.request.cas); } else { return test_binary_noop(); } return TEST_PASS; } static enum test_return test_binary_set(void) { return test_binary_set_impl("test_binary_set", PROTOCOL_BINARY_CMD_SET); } static enum test_return test_binary_setq(void) { return test_binary_set_impl("test_binary_setq", PROTOCOL_BINARY_CMD_SETQ); } static enum test_return test_binary_add_impl(const char *key, uint8_t cmd) { uint64_t value = 0xdeadbeefdeadcafe; union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } send, receive; size_t len = storage_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), &value, sizeof(value), 0, 0); /* Add should only work the first time */ int ii; for (ii = 0; ii < 10; ++ii) { safe_send(send.bytes, len, false); if (ii == 0) { if (cmd == PROTOCOL_BINARY_CMD_ADD) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); } } else { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS); } } return TEST_PASS; } static enum test_return test_binary_add(void) { return test_binary_add_impl("test_binary_add", PROTOCOL_BINARY_CMD_ADD); } static enum test_return test_binary_addq(void) { return test_binary_add_impl("test_binary_addq", PROTOCOL_BINARY_CMD_ADDQ); } static enum test_return test_binary_replace_impl(const char* key, uint8_t cmd) { uint64_t value = 0xdeadbeefdeadcafe; union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } send, receive; size_t len = storage_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), &value, sizeof(value), 0, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT); len = storage_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_ADD, key, strlen(key), &value, sizeof(value), 0, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_ADD, PROTOCOL_BINARY_RESPONSE_SUCCESS); len = storage_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), &value, sizeof(value), 0, 0); int ii; for (ii = 0; ii < 10; ++ii) { safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_REPLACE) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_REPLACE, PROTOCOL_BINARY_RESPONSE_SUCCESS); } } if (cmd == PROTOCOL_BINARY_CMD_REPLACEQ) { test_binary_noop(); } return TEST_PASS; } static enum test_return test_binary_replace(void) { return test_binary_replace_impl("test_binary_replace", PROTOCOL_BINARY_CMD_REPLACE); } static enum test_return test_binary_replaceq(void) { return test_binary_replace_impl("test_binary_replaceq", PROTOCOL_BINARY_CMD_REPLACEQ); } static enum test_return test_binary_delete_impl(const char *key, uint8_t cmd) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } send, receive; size_t len = raw_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), NULL, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT); len = storage_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_ADD, key, strlen(key), NULL, 0, 0, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_ADD, PROTOCOL_BINARY_RESPONSE_SUCCESS); len = raw_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), NULL, 0); safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_DELETE) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_DELETE, PROTOCOL_BINARY_RESPONSE_SUCCESS); } safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT); return TEST_PASS; } static enum test_return test_binary_delete(void) { return test_binary_delete_impl("test_binary_delete", PROTOCOL_BINARY_CMD_DELETE); } static enum test_return test_binary_deleteq(void) { return test_binary_delete_impl("test_binary_deleteq", PROTOCOL_BINARY_CMD_DELETEQ); } static enum test_return test_binary_get_impl(const char *key, uint8_t cmd) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } send, receive; uint32_t expiration = htonl(3600); size_t extlen = 0; if (cmd == PROTOCOL_BINARY_CMD_GAT || cmd == PROTOCOL_BINARY_CMD_GATK) extlen = sizeof(expiration); size_t len = ext_command(send.bytes, sizeof(send.bytes), cmd, extlen ? &expiration : NULL, extlen, key, strlen(key), NULL, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT); len = storage_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_ADD, key, strlen(key), NULL, 0, 0, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_ADD, PROTOCOL_BINARY_RESPONSE_SUCCESS); /* run a little pipeline test ;-) */ len = 0; int ii; for (ii = 0; ii < 10; ++ii) { union { protocol_binary_request_no_extras request; char bytes[1024]; } temp; size_t l = ext_command(temp.bytes, sizeof(temp.bytes), cmd, extlen ? &expiration : NULL, extlen, key, strlen(key), NULL, 0); memcpy(send.bytes + len, temp.bytes, l); len += l; } safe_send(send.bytes, len, false); for (ii = 0; ii < 10; ++ii) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); } return TEST_PASS; } static enum test_return test_binary_get(void) { return test_binary_get_impl("test_binary_get", PROTOCOL_BINARY_CMD_GET); } static enum test_return test_binary_getk(void) { return test_binary_get_impl("test_binary_getk", PROTOCOL_BINARY_CMD_GETK); } static enum test_return test_binary_gat(void) { return test_binary_get_impl("test_binary_gat", PROTOCOL_BINARY_CMD_GAT); } static enum test_return test_binary_gatk(void) { return test_binary_get_impl("test_binary_gatk", PROTOCOL_BINARY_CMD_GATK); } static enum test_return test_binary_getq_impl(const char *key, uint8_t cmd) { const char *missing = "test_binary_getq_missing"; union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } send, temp, receive; uint32_t expiration = htonl(3600); size_t extlen = 0; if (cmd == PROTOCOL_BINARY_CMD_GATQ || cmd == PROTOCOL_BINARY_CMD_GATKQ) extlen = sizeof(expiration); size_t len = storage_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_ADD, key, strlen(key), NULL, 0, 0, 0); size_t len2 = ext_command(temp.bytes, sizeof(temp.bytes), cmd, extlen ? &expiration : NULL, extlen, missing, strlen(missing), NULL, 0); /* I need to change the first opaque so that I can separate the two * return packets */ temp.request.message.header.request.opaque = 0xfeedface; memcpy(send.bytes + len, temp.bytes, len2); len += len2; len2 = ext_command(temp.bytes, sizeof(temp.bytes), cmd, extlen ? &expiration : NULL, extlen, key, strlen(key), NULL, 0); memcpy(send.bytes + len, temp.bytes, len2); len += len2; safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_ADD, PROTOCOL_BINARY_RESPONSE_SUCCESS); /* The first GETQ shouldn't return anything */ safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); return TEST_PASS; } static enum test_return test_binary_getq(void) { return test_binary_getq_impl("test_binary_getq", PROTOCOL_BINARY_CMD_GETQ); } static enum test_return test_binary_getkq(void) { return test_binary_getq_impl("test_binary_getkq", PROTOCOL_BINARY_CMD_GETKQ); } static enum test_return test_binary_gatq(void) { return test_binary_getq_impl("test_binary_gatq", PROTOCOL_BINARY_CMD_GATQ); } static enum test_return test_binary_gatkq(void) { return test_binary_getq_impl("test_binary_gatkq", PROTOCOL_BINARY_CMD_GATKQ); } static enum test_return test_binary_incr_impl(const char* key, uint8_t cmd) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response_header; protocol_binary_response_incr response; char bytes[1024]; } send, receive; size_t len = arithmetic_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), 1, 0, 0); int ii; for (ii = 0; ii < 10; ++ii) { safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_INCREMENT) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response_header, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); assert(ntohll(receive.response.message.body.value) == ii); } } if (cmd == PROTOCOL_BINARY_CMD_INCREMENTQ) { test_binary_noop(); } return TEST_PASS; } static enum test_return test_binary_incr(void) { return test_binary_incr_impl("test_binary_incr", PROTOCOL_BINARY_CMD_INCREMENT); } static enum test_return test_binary_incrq(void) { return test_binary_incr_impl("test_binary_incrq", PROTOCOL_BINARY_CMD_INCREMENTQ); } static enum test_return test_binary_decr_impl(const char* key, uint8_t cmd) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response_header; protocol_binary_response_decr response; char bytes[1024]; } send, receive; size_t len = arithmetic_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), 1, 9, 0); int ii; for (ii = 9; ii >= 0; --ii) { safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_DECREMENT) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response_header, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); assert(ntohll(receive.response.message.body.value) == ii); } } /* decr on 0 should not wrap */ safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_DECREMENT) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response_header, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); assert(ntohll(receive.response.message.body.value) == 0); } else { test_binary_noop(); } return TEST_PASS; } static enum test_return test_binary_decr(void) { return test_binary_decr_impl("test_binary_decr", PROTOCOL_BINARY_CMD_DECREMENT); } static enum test_return test_binary_decrq(void) { return test_binary_decr_impl("test_binary_decrq", PROTOCOL_BINARY_CMD_DECREMENTQ); } static enum test_return test_binary_version(void) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } buffer; size_t len = raw_command(buffer.bytes, sizeof(buffer.bytes), PROTOCOL_BINARY_CMD_VERSION, NULL, 0, NULL, 0); safe_send(buffer.bytes, len, false); safe_recv_packet(buffer.bytes, sizeof(buffer.bytes)); validate_response_header(&buffer.response, PROTOCOL_BINARY_CMD_VERSION, PROTOCOL_BINARY_RESPONSE_SUCCESS); return TEST_PASS; } static enum test_return test_binary_flush_impl(const char *key, uint8_t cmd) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } send, receive; size_t len = storage_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_ADD, key, strlen(key), NULL, 0, 0, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_ADD, PROTOCOL_BINARY_RESPONSE_SUCCESS); len = flush_command(send.bytes, sizeof(send.bytes), cmd, 2, true); safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_FLUSH) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); } len = raw_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_GET, key, strlen(key), NULL, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_GET, PROTOCOL_BINARY_RESPONSE_SUCCESS); sleep(2); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_GET, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT); int ii; for (ii = 0; ii < 2; ++ii) { len = storage_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_ADD, key, strlen(key), NULL, 0, 0, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_ADD, PROTOCOL_BINARY_RESPONSE_SUCCESS); len = flush_command(send.bytes, sizeof(send.bytes), cmd, 0, ii == 0); safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_FLUSH) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); } len = raw_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_GET, key, strlen(key), NULL, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_GET, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT); } return TEST_PASS; } static enum test_return test_binary_flush(void) { return test_binary_flush_impl("test_binary_flush", PROTOCOL_BINARY_CMD_FLUSH); } static enum test_return test_binary_flushq(void) { return test_binary_flush_impl("test_binary_flushq", PROTOCOL_BINARY_CMD_FLUSHQ); } static enum test_return test_binary_concat_impl(const char *key, uint8_t cmd) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } send, receive; const char *value = "world"; size_t len = raw_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), value, strlen(value)); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_NOT_STORED); len = storage_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_ADD, key, strlen(key), value, strlen(value), 0, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_ADD, PROTOCOL_BINARY_RESPONSE_SUCCESS); len = raw_command(send.bytes, sizeof(send.bytes), cmd, key, strlen(key), value, strlen(value)); safe_send(send.bytes, len, false); if (cmd == PROTOCOL_BINARY_CMD_APPEND || cmd == PROTOCOL_BINARY_CMD_PREPEND) { safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, cmd, PROTOCOL_BINARY_RESPONSE_SUCCESS); } else { len = raw_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_NOOP, NULL, 0, NULL, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_NOOP, PROTOCOL_BINARY_RESPONSE_SUCCESS); } len = raw_command(send.bytes, sizeof(send.bytes), PROTOCOL_BINARY_CMD_GETK, key, strlen(key), NULL, 0); safe_send(send.bytes, len, false); safe_recv_packet(receive.bytes, sizeof(receive.bytes)); validate_response_header(&receive.response, PROTOCOL_BINARY_CMD_GETK, PROTOCOL_BINARY_RESPONSE_SUCCESS); assert(receive.response.message.header.response.keylen == strlen(key)); assert(receive.response.message.header.response.bodylen == (strlen(key) + 2*strlen(value) + 4)); char *ptr = receive.bytes; ptr += sizeof(receive.response); ptr += 4; assert(memcmp(ptr, key, strlen(key)) == 0); ptr += strlen(key); assert(memcmp(ptr, value, strlen(value)) == 0); ptr += strlen(value); assert(memcmp(ptr, value, strlen(value)) == 0); return TEST_PASS; } static enum test_return test_binary_append(void) { return test_binary_concat_impl("test_binary_append", PROTOCOL_BINARY_CMD_APPEND); } static enum test_return test_binary_prepend(void) { return test_binary_concat_impl("test_binary_prepend", PROTOCOL_BINARY_CMD_PREPEND); } static enum test_return test_binary_appendq(void) { return test_binary_concat_impl("test_binary_appendq", PROTOCOL_BINARY_CMD_APPENDQ); } static enum test_return test_binary_prependq(void) { return test_binary_concat_impl("test_binary_prependq", PROTOCOL_BINARY_CMD_PREPENDQ); } static enum test_return test_binary_stat(void) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } buffer; size_t len = raw_command(buffer.bytes, sizeof(buffer.bytes), PROTOCOL_BINARY_CMD_STAT, NULL, 0, NULL, 0); safe_send(buffer.bytes, len, false); do { safe_recv_packet(buffer.bytes, sizeof(buffer.bytes)); validate_response_header(&buffer.response, PROTOCOL_BINARY_CMD_STAT, PROTOCOL_BINARY_RESPONSE_SUCCESS); } while (buffer.response.message.header.response.keylen != 0); return TEST_PASS; } static enum test_return test_binary_illegal(void) { uint8_t cmd = 0x25; while (cmd != 0x00) { union { protocol_binary_request_no_extras request; protocol_binary_response_no_extras response; char bytes[1024]; } buffer; size_t len = raw_command(buffer.bytes, sizeof(buffer.bytes), cmd, NULL, 0, NULL, 0); safe_send(buffer.bytes, len, false); safe_recv_packet(buffer.bytes, sizeof(buffer.bytes)); validate_response_header(&buffer.response, cmd, PROTOCOL_BINARY_RESPONSE_UNKNOWN_COMMAND); ++cmd; } return TEST_PASS; } volatile bool hickup_thread_running; static void *binary_hickup_recv_verification_thread(void *arg) { protocol_binary_response_no_extras *response = malloc(65*1024); if (response != NULL) { while (safe_recv_packet(response, 65*1024)) { /* Just validate the packet format */ validate_response_header(response, response->message.header.response.opcode, response->message.header.response.status); } free(response); } hickup_thread_running = false; allow_closed_read = false; return NULL; } static enum test_return test_binary_pipeline_hickup_chunk(void *buffer, size_t buffersize) { off_t offset = 0; char *key[256]; uint64_t value = 0xfeedfacedeadbeef; while (hickup_thread_running && offset + sizeof(protocol_binary_request_no_extras) < buffersize) { union { protocol_binary_request_no_extras request; char bytes[65 * 1024]; } command; uint8_t cmd = (uint8_t)(rand() & 0xff); size_t len; size_t keylen = (rand() % 250) + 1; switch (cmd) { case PROTOCOL_BINARY_CMD_ADD: case PROTOCOL_BINARY_CMD_ADDQ: case PROTOCOL_BINARY_CMD_REPLACE: case PROTOCOL_BINARY_CMD_REPLACEQ: case PROTOCOL_BINARY_CMD_SET: case PROTOCOL_BINARY_CMD_SETQ: len = storage_command(command.bytes, sizeof(command.bytes), cmd, key, keylen , &value, sizeof(value), 0, 0); break; case PROTOCOL_BINARY_CMD_APPEND: case PROTOCOL_BINARY_CMD_APPENDQ: case PROTOCOL_BINARY_CMD_PREPEND: case PROTOCOL_BINARY_CMD_PREPENDQ: len = raw_command(command.bytes, sizeof(command.bytes), cmd, key, keylen, &value, sizeof(value)); break; case PROTOCOL_BINARY_CMD_FLUSH: case PROTOCOL_BINARY_CMD_FLUSHQ: len = raw_command(command.bytes, sizeof(command.bytes), cmd, NULL, 0, NULL, 0); break; case PROTOCOL_BINARY_CMD_NOOP: len = raw_command(command.bytes, sizeof(command.bytes), cmd, NULL, 0, NULL, 0); break; case PROTOCOL_BINARY_CMD_DELETE: case PROTOCOL_BINARY_CMD_DELETEQ: len = raw_command(command.bytes, sizeof(command.bytes), cmd, key, keylen, NULL, 0); break; case PROTOCOL_BINARY_CMD_DECREMENT: case PROTOCOL_BINARY_CMD_DECREMENTQ: case PROTOCOL_BINARY_CMD_INCREMENT: case PROTOCOL_BINARY_CMD_INCREMENTQ: len = arithmetic_command(command.bytes, sizeof(command.bytes), cmd, key, keylen, 1, 0, 0); break; case PROTOCOL_BINARY_CMD_VERSION: len = raw_command(command.bytes, sizeof(command.bytes), PROTOCOL_BINARY_CMD_VERSION, NULL, 0, NULL, 0); break; case PROTOCOL_BINARY_CMD_GET: case PROTOCOL_BINARY_CMD_GETK: case PROTOCOL_BINARY_CMD_GETKQ: case PROTOCOL_BINARY_CMD_GETQ: len = raw_command(command.bytes, sizeof(command.bytes), cmd, key, keylen, NULL, 0); break; case PROTOCOL_BINARY_CMD_TOUCH: case PROTOCOL_BINARY_CMD_GAT: case PROTOCOL_BINARY_CMD_GATQ: case PROTOCOL_BINARY_CMD_GATK: case PROTOCOL_BINARY_CMD_GATKQ: len = touch_command(command.bytes, sizeof(command.bytes), cmd, key, keylen, 10); break; case PROTOCOL_BINARY_CMD_STAT: len = raw_command(command.bytes, sizeof(command.bytes), PROTOCOL_BINARY_CMD_STAT, NULL, 0, NULL, 0); break; case PROTOCOL_BINARY_CMD_SASL_LIST_MECHS: case PROTOCOL_BINARY_CMD_SASL_AUTH: case PROTOCOL_BINARY_CMD_SASL_STEP: /* Ignoring SASL */ case PROTOCOL_BINARY_CMD_QUITQ: case PROTOCOL_BINARY_CMD_QUIT: /* I don't want to pass on the quit commands ;-) */ cmd |= 0xf0; /* FALLTHROUGH */ default: len = raw_command(command.bytes, sizeof(command.bytes), cmd, NULL, 0, NULL, 0); } if ((len + offset) < buffersize) { memcpy(((char*)buffer) + offset, command.bytes, len); offset += len; } else { break; } } safe_send(buffer, offset, true); return TEST_PASS; } static enum test_return test_binary_pipeline_hickup(void) { size_t buffersize = 65 * 1024; void *buffer = malloc(buffersize); int ii; pthread_t tid; int ret; allow_closed_read = true; hickup_thread_running = true; if ((ret = pthread_create(&tid, NULL, binary_hickup_recv_verification_thread, NULL)) != 0) { fprintf(stderr, "Can't create thread: %s\n", strerror(ret)); free(buffer); return TEST_FAIL; } /* Allow the thread to start */ usleep(250); srand((int)time(NULL)); for (ii = 0; ii < 2; ++ii) { test_binary_pipeline_hickup_chunk(buffer, buffersize); } /* send quitq to shut down the read thread ;-) */ size_t len = raw_command(buffer, buffersize, PROTOCOL_BINARY_CMD_QUITQ, NULL, 0, NULL, 0); safe_send(buffer, len, false); pthread_join(tid, NULL); free(buffer); return TEST_PASS; } static enum test_return test_issue_101(void) { enum { max = 2 }; enum test_return ret = TEST_PASS; struct conn *conns[max]; int ii = 0; pid_t child = 0; if (getenv("SKIP_TEST_101") != NULL) { return TEST_SKIP; } const char *command = "stats\r\nstats\r\nstats\r\nstats\r\nstats\r\n"; size_t cmdlen = strlen(command); server_pid = start_server(&port, false, 1000); for (ii = 0; ii < max; ++ii) { conns[ii] = NULL; conns[ii] = connect_server("127.0.0.1", port, true, enable_ssl); assert(conns[ii]); assert(conns[ii]->sock > 0); } /* Send command on the connection until it blocks */ for (ii = 0; ii < max; ++ii) { bool more = true; do { ssize_t err = conns[ii]->write(conns[ii], command, cmdlen); if (err == -1) { switch (errno) { case EINTR: break; case ENOMEM: case EWOULDBLOCK: more = false; break; default: ret = TEST_FAIL; goto cleanup; } } } while (more); } child = fork(); if (child == (pid_t)-1) { abort(); } else if (child > 0) { int stat; pid_t c; while ((c = waitpid(child, &stat, 0)) == (pid_t)-1 && errno == EINTR); assert(c == child); assert(stat == 0); } else { con = connect_server("127.0.0.1", port, false, enable_ssl); assert(con); ret = test_binary_noop(); exit(0); } cleanup: /* close all connections */ for (ii = 0; ii < max; ++ii) { struct conn *c = conns[ii]; if (c == NULL) continue; #ifdef TLS if (c->ssl) { SSL_shutdown(c->ssl); SSL_free(c->ssl); } if (c->ssl_ctx) SSL_CTX_free(c->ssl_ctx); #endif if (c->sock > 0) close(c->sock); free(conns[ii]); conns[ii] = NULL; } assert(kill(server_pid, SIGTERM) == 0); return ret; } typedef enum test_return (*TEST_FUNC)(void); struct testcase { const char *description; TEST_FUNC function; }; struct testcase testcases[] = { { "cache_create", cache_create_test }, { "cache_constructor", cache_constructor_test }, { "cache_constructor_fail", cache_fail_constructor_test }, { "cache_destructor", cache_destructor_test }, { "cache_reuse", cache_reuse_test }, { "cache_redzone", cache_redzone_test }, { "stats_prefix_find", test_stats_prefix_find }, { "stats_prefix_record_get", test_stats_prefix_record_get }, { "stats_prefix_record_delete", test_stats_prefix_record_delete }, { "stats_prefix_record_set", test_stats_prefix_record_set }, { "stats_prefix_dump", test_stats_prefix_dump }, { "issue_161", test_issue_161 }, { "strtol", test_safe_strtol }, { "strtoll", test_safe_strtoll }, { "strtoul", test_safe_strtoul }, { "strtoull", test_safe_strtoull }, { "issue_44", test_issue_44 }, { "vperror", test_vperror }, { "issue_101", test_issue_101 }, /* The following tests all run towards the same server */ { "start_server", start_memcached_server }, { "issue_92", test_issue_92 }, { "issue_102", test_issue_102 }, { "binary_noop", test_binary_noop }, { "binary_quit", test_binary_quit }, { "binary_quitq", test_binary_quitq }, { "binary_set", test_binary_set }, { "binary_setq", test_binary_setq }, { "binary_add", test_binary_add }, { "binary_addq", test_binary_addq }, { "binary_replace", test_binary_replace }, { "binary_replaceq", test_binary_replaceq }, { "binary_delete", test_binary_delete }, { "binary_deleteq", test_binary_deleteq }, { "binary_get", test_binary_get }, { "binary_getq", test_binary_getq }, { "binary_getk", test_binary_getk }, { "binary_getkq", test_binary_getkq }, { "binary_gat", test_binary_gat }, { "binary_gatq", test_binary_gatq }, { "binary_gatk", test_binary_gatk }, { "binary_gatkq", test_binary_gatkq }, { "binary_incr", test_binary_incr }, { "binary_incrq", test_binary_incrq }, { "binary_decr", test_binary_decr }, { "binary_decrq", test_binary_decrq }, { "binary_version", test_binary_version }, { "binary_flush", test_binary_flush }, { "binary_flushq", test_binary_flushq }, { "binary_append", test_binary_append }, { "binary_appendq", test_binary_appendq }, { "binary_prepend", test_binary_prepend }, { "binary_prependq", test_binary_prependq }, { "binary_stat", test_binary_stat }, { "binary_illegal", test_binary_illegal }, { "binary_pipeline_hickup", test_binary_pipeline_hickup }, { "shutdown", shutdown_memcached_server }, { "stop_server", stop_memcached_server }, { NULL, NULL } }; /* Stub out function defined in memcached.c */ void STATS_LOCK(void); void STATS_UNLOCK(void); void STATS_LOCK(void) {} void STATS_UNLOCK(void) {} int main(int argc, char **argv) { int exitcode = 0; int ii = 0, num_cases = 0; #ifdef TLS if (getenv("SSL_TEST") != NULL) { SSLeay_add_ssl_algorithms(); SSL_load_error_strings(); enable_ssl = true; } #endif /* Stats prefix test is sensitive to the choice of hash function */ hash_init(JENKINS_HASH); stats_prefix_init(':'); for (num_cases = 0; testcases[num_cases].description; num_cases++) { /* Just counting */ } printf("1..%d\n", num_cases); for (ii = 0; testcases[ii].description != NULL; ++ii) { fflush(stdout); #ifndef DEBUG /* the test program shouldn't run longer than 10 minutes... */ alarm(600); #endif enum test_return ret = testcases[ii].function(); if (ret == TEST_SKIP) { fprintf(stdout, "ok # SKIP %d - %s\n", ii + 1, testcases[ii].description); } else if (ret == TEST_PASS) { fprintf(stdout, "ok %d - %s\n", ii + 1, testcases[ii].description); } else { fprintf(stdout, "not ok %d - %s\n", ii + 1, testcases[ii].description); exitcode = 1; } fflush(stdout); } return exitcode; } memcached-1.5.22/logger.h0000664000175000017500000001154313575022500012065 00000000000000/* logging functions */ #ifndef LOGGER_H #define LOGGER_H #include "bipbuffer.h" /* TODO: starttime tunable */ #define LOGGER_BUF_SIZE 1024 * 64 #define LOGGER_WATCHER_BUF_SIZE 1024 * 256 #define LOGGER_ENTRY_MAX_SIZE 2048 #define GET_LOGGER() ((logger *) pthread_getspecific(logger_key)); /* Inlined from memcached.h - should go into sub header */ typedef unsigned int rel_time_t; enum log_entry_type { LOGGER_ASCII_CMD = 0, LOGGER_EVICTION, LOGGER_ITEM_GET, LOGGER_ITEM_STORE, LOGGER_CRAWLER_STATUS, LOGGER_SLAB_MOVE, #ifdef EXTSTORE LOGGER_EXTSTORE_WRITE, LOGGER_COMPACT_START, LOGGER_COMPACT_ABORT, LOGGER_COMPACT_READ_START, LOGGER_COMPACT_READ_END, LOGGER_COMPACT_END, LOGGER_COMPACT_FRAGINFO, #endif }; enum log_entry_subtype { LOGGER_TEXT_ENTRY = 0, LOGGER_EVICTION_ENTRY, LOGGER_ITEM_GET_ENTRY, LOGGER_ITEM_STORE_ENTRY, #ifdef EXTSTORE LOGGER_EXT_WRITE_ENTRY, #endif }; enum logger_ret_type { LOGGER_RET_OK = 0, LOGGER_RET_NOSPACE, LOGGER_RET_ERR }; enum logger_parse_entry_ret { LOGGER_PARSE_ENTRY_OK = 0, LOGGER_PARSE_ENTRY_FULLBUF, LOGGER_PARSE_ENTRY_FAILED }; typedef const struct { enum log_entry_subtype subtype; int reqlen; uint16_t eflags; char *format; } entry_details; /* log entry intermediary structures */ struct logentry_eviction { long long int exptime; uint32_t latime; uint16_t it_flags; uint8_t nkey; uint8_t clsid; char key[]; }; #ifdef EXTSTORE struct logentry_ext_write { long long int exptime; uint32_t latime; uint16_t it_flags; uint8_t nkey; uint8_t clsid; uint8_t bucket; char key[]; }; #endif struct logentry_item_get { uint8_t was_found; uint8_t nkey; uint8_t clsid; int sfd; char key[]; }; struct logentry_item_store { int status; int cmd; rel_time_t ttl; uint8_t nkey; uint8_t clsid; int sfd; char key[]; }; /* end intermediary structures */ /* WARNING: cuddled items aren't compatible with warm restart. more code * necessary to ensure log streams are all flushed/processed before stopping */ typedef struct _logentry { enum log_entry_subtype event; uint8_t pad; uint16_t eflags; uint64_t gid; struct timeval tv; /* not monotonic! */ int size; union { char end; } data[]; } logentry; #define LOG_SYSEVENTS (1<<1) /* threads start/stop/working */ #define LOG_FETCHERS (1<<2) /* get/gets/etc */ #define LOG_MUTATIONS (1<<3) /* set/append/incr/etc */ #define LOG_SYSERRORS (1<<4) /* malloc/etc errors */ #define LOG_CONNEVENTS (1<<5) /* new client, closed, etc */ #define LOG_EVICTIONS (1<<6) /* details of evicted items */ #define LOG_STRICT (1<<7) /* block worker instead of drop */ #define LOG_RAWCMDS (1<<9) /* raw ascii commands */ typedef struct _logger { struct _logger *prev; struct _logger *next; pthread_mutex_t mutex; /* guard for this + *buf */ uint64_t written; /* entries written to the buffer */ uint64_t dropped; /* entries dropped */ uint64_t blocked; /* times blocked instead of dropped */ uint16_t fetcher_ratio; /* log one out of every N fetches */ uint16_t mutation_ratio; /* log one out of every N mutations */ uint16_t eflags; /* flags this logger should log */ bipbuf_t *buf; const entry_details *entry_map; } logger; enum logger_watcher_type { LOGGER_WATCHER_STDERR = 0, LOGGER_WATCHER_CLIENT = 1 }; typedef struct { void *c; /* original connection structure. still with source thread attached */ int sfd; /* client fd */ int id; /* id number for watcher list */ uint64_t skipped; /* lines skipped since last successful print */ bool failed_flush; /* recently failed to write out (EAGAIN), wait before retry */ enum logger_watcher_type t; /* stderr, client, syslog, etc */ uint16_t eflags; /* flags we are interested in */ bipbuf_t *buf; /* per-watcher output buffer */ } logger_watcher; struct logger_stats { uint64_t worker_dropped; uint64_t worker_written; uint64_t watcher_skipped; uint64_t watcher_sent; }; extern pthread_key_t logger_key; /* public functions */ void logger_init(void); void logger_stop(void); logger *logger_create(void); #define LOGGER_LOG(l, flag, type, ...) \ do { \ logger *myl = l; \ if (l == NULL) \ myl = GET_LOGGER(); \ if (myl->eflags & flag) \ logger_log(myl, type, __VA_ARGS__); \ } while (0) enum logger_ret_type logger_log(logger *l, const enum log_entry_type event, const void *entry, ...); enum logger_add_watcher_ret { LOGGER_ADD_WATCHER_TOO_MANY = 0, LOGGER_ADD_WATCHER_OK, LOGGER_ADD_WATCHER_FAILED }; enum logger_add_watcher_ret logger_add_watcher(void *c, const int sfd, uint16_t f); /* functions used by restart system */ uint64_t logger_get_gid(void); void logger_set_gid(uint64_t gid); #endif memcached-1.5.22/memcached.spec0000664000175000017500000001334713615454042013230 00000000000000%bcond_with arm_crc32 %bcond_with extstore %bcond_with seccomp %bcond_with sasl %bcond_with sasl_pwdb %bcond_with dtrace %bcond_with 64bit %bcond_without option_checking %bcond_without coverage %bcond_without docs # Set with_systemd on distros that use it, so we can install the service # file, otherwise the sysvinit script will be installed %if 0%{?fedora} >= 14 || 0%{?rhel} >= 7 || 0%{?suse_version} >= 1210 %global with_systemd 1 BuildRequires: systemd-units # Disable some systemd safety features on OSes without a new enough systemd # (new enough is systemd >= 233) %if 0%{?fedora} < 26 || 0%{?rhel} > 0 %global safer_systemd 0 %else %global safer_systemd 1 %endif %else %global with_systemd 0 %endif Name: memcached Version: 1.5.22 Release: 1%{?dist} Summary: High Performance, Distributed Memory Object Cache Group: System Environment/Daemons License: BSD URL: https://memcached.org Source0: https://memcached.org/files/%{name}-%{version}.tar.gz Source1: memcached.sysconfig Source2: memcached.service Source3: memcached@.service BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildRequires: libevent-devel BuildRequires: perl(Test::More) BuildRequires: /usr/bin/prove Requires: initscripts %if %{with_systemd} Requires(post): systemd-units Requires(preun): systemd-units Requires(postun): systemd-units %else Requires(post): /sbin/chkconfig Requires(preun): /sbin/chkconfig, /sbin/service Requires(postun): /sbin/service %endif %description memcached is a high-performance, distributed memory object caching system, generic in nature, but intended for use in speeding up dynamic web applications by alleviating database load. %prep %setup -q -n %{name}-%{version} %build %configure \ %{?with_arm_crc32:--enable-arm-crc32} \ %{?with_extstore:--enable-extstore} \ %{?with_seccomp:--enable-seccomp} \ %{?with_sasl:--enable-sasl} \ %{?with_sasl_pwdb:--enable-pwdb} \ %{?with_dtrace:--enable-dtrace} \ %{?with_64bit:--enable-64bit} \ %{!?with_option_checking:--disable-option-checking} %{!?with_coverage:--disable-coverage} \ %{!?with_docs:--disable-docs} make %{?_smp_mflags} %check make test %install rm -rf %{buildroot} make install DESTDIR=%{buildroot} # remove memcached-debug rm -f %{buildroot}/%{_bindir}/%{name}-debug # Perl script for monitoring memcached install -Dp -m0755 scripts/memcached-tool %{buildroot}%{_bindir}/%{name}-tool # Init script %if %{with_systemd} install -Dp -m0755 scripts/memcached.service %{buildroot}%{_unitdir}/%{name}.service install -Dp -m0755 scripts/memcached@.service %{buildroot}%{_unitdir}/%{name}@.service if [ %{safer_systemd} -gt 0 ]; then sed -e 's/^##safer##//g' -i %{buildroot}%{_unitdir}/%{name}.service %{buildroot}%{_unitdir}/%{name}@.service else sed -e 's/^##safer##/#/g' -i %{buildroot}%{_unitdir}/%{name}.service %{buildroot}%{_unitdir}/%{name}@.service fi %else install -Dp -m0755 scripts/memcached.sysv %{buildroot}%{_initrddir}/%{name} %endif # Default configs install -Dp -m0644 scripts/memcached.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/%{name} # pid directory mkdir -p %{buildroot}/%{_localstatedir}/run/%{name} %clean rm -rf %{buildroot} %post if [ $1 -eq 1 ]; then # Initial install %if %{with_systemd} /bin/systemctl daemon-reload >/dev/null 2>&1 || : %else /sbin/chkconfig --add %{name} %endif fi %preun if [ "$1" = 0 ] ; then # Removal, not upgrade %if %{with_systemd} /bin/systemctl --no-reload disable %{name}.service > /dev/null 2>&1 || : /bin/systemctl --no-reload disable %{name}@\*.service > /dev/null 2>&1 || : /bin/systemctl stop %{name}.service > /dev/null 2>&1 || : /bin/systemctl stop %{name}@\*.service > /dev/null 2>&1 || : %else /sbin/service %{name} stop > /dev/null 2&>1 || : /sbin/chkconfig --del %{name} %endif fi exit 0 %postun %if %{with_systemd} /bin/systemctl daemon-reload >/dev/null 2>&1 || : %endif # Don't auto-restart memcached on upgrade -- let user control when cache flushes # if [ "$1" -ge 1 ]; then # # upgrade, not install # %if %{with_systemd} # /bin/systemctl try-restart %{name}.service # /bin/systemctl try-restart %{name}@\*.service # %else # /sbin/service %named condrestart 2>/dev/null || : # %endif #fi exit 0 %files %defattr(-,root,root,-) %doc AUTHORS ChangeLog COPYING NEWS README.md doc/CONTRIBUTORS doc/*.txt %config(noreplace) %{_sysconfdir}/sysconfig/%{name} %dir %attr(750,nobody,nobody) %{_localstatedir}/run/%{name} %{_bindir}/%{name}-tool %{_bindir}/%{name} %{_mandir}/man1/%{name}.1* %{_includedir}/%{name} %if %{with_systemd} %{_unitdir}/%{name}.service %{_unitdir}/%{name}@.service %else %{_initrddir}/%{name} %endif %changelog * Wed Jul 5 2017 J. Grizzard - 1.4.39 - Add systemd-aware build - Add both static and instanced versions of memcached unit files * Mon Nov 2 2009 Dormando - 1.4.3-1 - Fix autogen more. * Sat Aug 29 2009 Dustin Sallings - 1.4.1-1 - Autogenerate the version number from tags. * Wed Jul 4 2007 Paul Lindner - 1.2.2-5 - Use /var/run/memcached/ directory to hold PID file * Sat May 12 2007 Paul Lindner - 1.2.2-4 - Remove tabs from spec file, rpmlint reports no more errors * Thu May 10 2007 Paul Lindner - 1.2.2-3 - Enable build-time regression tests - add dependency on initscripts - remove memcached-debug (not needed in dist) - above suggestions from Bernard Johnson * Mon May 7 2007 Paul Lindner - 1.2.2-2 - Tidiness improvements suggested by Ruben Kerkhof in bugzilla #238994 * Fri May 4 2007 Paul Lindner - 1.2.2-1 - Initial spec file created via rpmdev-newspec memcached-1.5.22/authfile.c0000664000175000017500000000607713611703161012410 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #include #include #include #include #include #include #include #include #include "authfile.h" #include "util.h" // TODO: frontend needs a refactor so this can avoid global objects. #define MAX_ENTRY_LEN 256 // Not supposed to be a huge database! #define MAX_ENTRIES 8 typedef struct auth_entry { char *user; size_t ulen; char *pass; size_t plen; } auth_t; auth_t main_auth_entries[MAX_ENTRIES]; int entry_cnt = 0; char *main_auth_data = NULL; enum authfile_ret authfile_load(const char *file) { struct stat sb; char *auth_data = NULL; auth_t auth_entries[MAX_ENTRIES]; if (stat(file, &sb) == -1) { return AUTHFILE_MISSING; } auth_data = calloc(1, sb.st_size); if (auth_data == NULL) { return AUTHFILE_OOM; } FILE *pwfile = fopen(file, "r"); if (pwfile == NULL) { // not strictly necessary but to be safe. free(auth_data); return AUTHFILE_OPENFAIL; } char *auth_cur = auth_data; auth_t *entry_cur = auth_entries; int used = 0; while ((fgets(auth_cur, MAX_ENTRY_LEN, pwfile)) != NULL) { int x; int found = 0; for (x = 0; x < MAX_ENTRY_LEN; x++) { if (!found && auth_cur[x] == ':') { entry_cur->user = auth_cur; entry_cur->ulen = x; entry_cur->pass = &auth_cur[x+1]; found = 1; } else if (found) { // Find end of password. if (auth_cur[x] == '\n' || auth_cur[x] == '\r' || auth_cur[x] == '\0') { entry_cur->plen = x - (entry_cur->ulen + 1); break; } } } // malformed line. if (!found) { (void)fclose(pwfile); free(auth_data); return AUTHFILE_MALFORMED; } // FIXME: no silent truncation. if (++used == MAX_ENTRIES) { break; } // EOF if (auth_cur[x] == '\0') break; auth_cur += x; entry_cur++; } // swap the main pointer out now, so if there's an error reloading we // don't break the existing authentication. if (main_auth_data != NULL) { free(main_auth_data); } entry_cnt = used; main_auth_data = auth_data; memcpy(main_auth_entries, auth_entries, sizeof(auth_entries)); (void)fclose(pwfile); return AUTHFILE_OK; } // if only loading the file could be this short... int authfile_check(const char *user, const char *pass) { size_t ulen = strlen(user); size_t plen = strlen(pass); for (int x = 0; x < entry_cnt; x++) { auth_t *e = &main_auth_entries[x]; if (ulen == e->ulen && plen == e->plen && safe_memcmp(user, e->user, e->ulen) && safe_memcmp(pass, e->pass, e->plen)) { return 1; } } return 0; } memcached-1.5.22/restart.h0000664000175000017500000000153313575022500012270 00000000000000#ifndef RESTART_H #define RESTART_H #define RESTART_TAG_MAXLEN 255 // Track the pointer size for restart fiddling. #if SIZEOF_VOID_P == 8 typedef uint64_t mc_ptr_t; #else typedef uint32_t mc_ptr_t; #endif enum restart_get_kv_ret { RESTART_OK=0, RESTART_NOTAG, RESTART_BADLINE, RESTART_DONE }; typedef int (*restart_check_cb)(const char *tag, void *ctx, void *data); typedef int (*restart_save_cb)(const char *tag, void *ctx, void *data); void restart_register(const char *tag, restart_check_cb ccb, restart_save_cb scb, void *data); void restart_set_kv(void *ctx, const char *key, const char *fmt, ...); enum restart_get_kv_ret restart_get_kv(void *ctx, char **key, char **val); bool restart_mmap_open(const size_t limit, const char *file, void **mem_base); void restart_mmap_close(void); unsigned int restart_fixup(void *old_base); #endif memcached-1.5.22/storage.c0000664000175000017500000005432013575022500012245 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #include "memcached.h" #ifdef EXTSTORE #include "storage.h" #include #include #include #include #define PAGE_BUCKET_DEFAULT 0 #define PAGE_BUCKET_COMPACT 1 #define PAGE_BUCKET_CHUNKED 2 #define PAGE_BUCKET_LOWTTL 3 /*** WRITE FLUSH THREAD ***/ static int storage_write(void *storage, const int clsid, const int item_age) { int did_moves = 0; struct lru_pull_tail_return it_info; it_info.it = NULL; lru_pull_tail(clsid, COLD_LRU, 0, LRU_PULL_RETURN_ITEM, 0, &it_info); /* Item is locked, and we have a reference to it. */ if (it_info.it == NULL) { return did_moves; } obj_io io; item *it = it_info.it; /* First, storage for the header object */ size_t orig_ntotal = ITEM_ntotal(it); uint32_t flags; if ((it->it_flags & ITEM_HDR) == 0 && (item_age == 0 || current_time - it->time > item_age)) { FLAGS_CONV(it, flags); item *hdr_it = do_item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, sizeof(item_hdr)); /* Run the storage write understanding the start of the item is dirty. * We will fill it (time/exptime/etc) from the header item on read. */ if (hdr_it != NULL) { int bucket = (it->it_flags & ITEM_CHUNKED) ? PAGE_BUCKET_CHUNKED : PAGE_BUCKET_DEFAULT; // Compress soon to expire items into similar pages. if (it->exptime - current_time < settings.ext_low_ttl) { bucket = PAGE_BUCKET_LOWTTL; } hdr_it->it_flags |= ITEM_HDR; io.len = orig_ntotal; io.mode = OBJ_IO_WRITE; // NOTE: when the item is read back in, the slab mover // may see it. Important to have refcount>=2 or ~ITEM_LINKED assert(it->refcount >= 2); // NOTE: write bucket vs free page bucket will disambiguate once // lowttl feature is better understood. if (extstore_write_request(storage, bucket, bucket, &io) == 0) { // cuddle the hash value into the time field so we don't have // to recalculate it. item *buf_it = (item *) io.buf; buf_it->time = it_info.hv; // copy from past the headers + time headers. // TODO: should be in items.c if (it->it_flags & ITEM_CHUNKED) { // Need to loop through the item and copy item_chunk *sch = (item_chunk *) ITEM_schunk(it); int remain = orig_ntotal; int copied = 0; // copy original header int hdrtotal = ITEM_ntotal(it) - it->nbytes; memcpy((char *)io.buf+STORE_OFFSET, (char *)it+STORE_OFFSET, hdrtotal - STORE_OFFSET); copied = hdrtotal; // copy data in like it were one large object. while (sch && remain) { assert(remain >= sch->used); memcpy((char *)io.buf+copied, sch->data, sch->used); // FIXME: use one variable? remain -= sch->used; copied += sch->used; sch = sch->next; } } else { memcpy((char *)io.buf+STORE_OFFSET, (char *)it+STORE_OFFSET, io.len-STORE_OFFSET); } // crc what we copied so we can do it sequentially. buf_it->it_flags &= ~ITEM_LINKED; buf_it->exptime = crc32c(0, (char*)io.buf+STORE_OFFSET, orig_ntotal-STORE_OFFSET); extstore_write(storage, &io); item_hdr *hdr = (item_hdr *) ITEM_data(hdr_it); hdr->page_version = io.page_version; hdr->page_id = io.page_id; hdr->offset = io.offset; // overload nbytes for the header it hdr_it->nbytes = it->nbytes; /* success! Now we need to fill relevant data into the new * header and replace. Most of this requires the item lock */ /* CAS gets set while linking. Copy post-replace */ item_replace(it, hdr_it, it_info.hv); ITEM_set_cas(hdr_it, ITEM_get_cas(it)); do_item_remove(hdr_it); did_moves = 1; LOGGER_LOG(NULL, LOG_EVICTIONS, LOGGER_EXTSTORE_WRITE, it, bucket); } else { /* Failed to write for some reason, can't continue. */ slabs_free(hdr_it, ITEM_ntotal(hdr_it), ITEM_clsid(hdr_it)); } } } do_item_remove(it); item_unlock(it_info.hv); return did_moves; } static pthread_t storage_write_tid; static pthread_mutex_t storage_write_plock; #define WRITE_SLEEP_MAX 1000000 #define WRITE_SLEEP_MIN 500 static void *storage_write_thread(void *arg) { void *storage = arg; // NOTE: ignoring overflow since that would take years of uptime in a // specific load pattern of never going to sleep. unsigned int backoff[MAX_NUMBER_OF_SLAB_CLASSES] = {0}; unsigned int counter = 0; useconds_t to_sleep = WRITE_SLEEP_MIN; logger *l = logger_create(); if (l == NULL) { fprintf(stderr, "Failed to allocate logger for storage compaction thread\n"); abort(); } pthread_mutex_lock(&storage_write_plock); while (1) { // cache per-loop to avoid calls to the slabs_clsid() search loop int min_class = slabs_clsid(settings.ext_item_size); bool do_sleep = true; counter++; if (to_sleep > WRITE_SLEEP_MAX) to_sleep = WRITE_SLEEP_MAX; for (int x = 0; x < MAX_NUMBER_OF_SLAB_CLASSES; x++) { bool did_move = false; bool mem_limit_reached = false; unsigned int chunks_free; int item_age; int target = settings.ext_free_memchunks[x]; if (min_class > x || (backoff[x] && (counter % backoff[x] != 0))) { // Long sleeps means we should retry classes sooner. if (to_sleep > WRITE_SLEEP_MIN * 10) backoff[x] /= 2; continue; } // Avoid extra slab lock calls during heavy writing. chunks_free = slabs_available_chunks(x, &mem_limit_reached, NULL); // storage_write() will fail and cut loop after filling write buffer. while (1) { // if we are low on chunks and no spare, push out early. if (chunks_free < target && mem_limit_reached) { item_age = 0; } else { item_age = settings.ext_item_age; } if (storage_write(storage, x, item_age)) { chunks_free++; // Allow stopping if we've done enough this loop did_move = true; do_sleep = false; if (to_sleep > WRITE_SLEEP_MIN) to_sleep /= 2; } else { break; } } if (!did_move) { backoff[x]++; } else if (backoff[x]) { backoff[x] /= 2; } } // flip lock so we can be paused or stopped pthread_mutex_unlock(&storage_write_plock); if (do_sleep) { usleep(to_sleep); to_sleep *= 2; } pthread_mutex_lock(&storage_write_plock); } return NULL; } // TODO // logger needs logger_destroy() to exist/work before this is safe. /*int stop_storage_write_thread(void) { int ret; pthread_mutex_lock(&lru_maintainer_lock); do_run_lru_maintainer_thread = 0; pthread_mutex_unlock(&lru_maintainer_lock); // WAKEUP SIGNAL if ((ret = pthread_join(lru_maintainer_tid, NULL)) != 0) { fprintf(stderr, "Failed to stop LRU maintainer thread: %s\n", strerror(ret)); return -1; } settings.lru_maintainer_thread = false; return 0; }*/ void storage_write_pause(void) { pthread_mutex_lock(&storage_write_plock); } void storage_write_resume(void) { pthread_mutex_unlock(&storage_write_plock); } int start_storage_write_thread(void *arg) { int ret; pthread_mutex_init(&storage_write_plock, NULL); if ((ret = pthread_create(&storage_write_tid, NULL, storage_write_thread, arg)) != 0) { fprintf(stderr, "Can't create storage_write thread: %s\n", strerror(ret)); return -1; } return 0; } /*** COMPACTOR ***/ /* Fetch stats from the external storage system and decide to compact. * If we're more than half full, start skewing how aggressively to run * compaction, up to a desired target when all pages are full. */ static int storage_compact_check(void *storage, logger *l, uint32_t *page_id, uint64_t *page_version, uint64_t *page_size, bool *drop_unread) { struct extstore_stats st; int x; double rate; uint64_t frag_limit; uint64_t low_version = ULLONG_MAX; uint64_t lowest_version = ULLONG_MAX; unsigned int low_page = 0; unsigned int lowest_page = 0; extstore_get_stats(storage, &st); if (st.pages_used == 0) return 0; // lets pick a target "wasted" value and slew. if (st.pages_free > settings.ext_compact_under) return 0; *drop_unread = false; // the number of free pages reduces the configured frag limit // this allows us to defrag early if pages are very empty. rate = 1.0 - ((double)st.pages_free / st.page_count); rate *= settings.ext_max_frag; frag_limit = st.page_size * rate; LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_FRAGINFO, NULL, rate, frag_limit); st.page_data = calloc(st.page_count, sizeof(struct extstore_page_data)); extstore_get_page_data(storage, &st); // find oldest page by version that violates the constraint for (x = 0; x < st.page_count; x++) { if (st.page_data[x].version == 0 || st.page_data[x].bucket == PAGE_BUCKET_LOWTTL) continue; if (st.page_data[x].version < lowest_version) { lowest_page = x; lowest_version = st.page_data[x].version; } if (st.page_data[x].bytes_used < frag_limit) { if (st.page_data[x].version < low_version) { low_page = x; low_version = st.page_data[x].version; } } } *page_size = st.page_size; free(st.page_data); // we have a page + version to attempt to reclaim. if (low_version != ULLONG_MAX) { *page_id = low_page; *page_version = low_version; return 1; } else if (lowest_version != ULLONG_MAX && settings.ext_drop_unread && st.pages_free <= settings.ext_drop_under) { // nothing matched the frag rate barrier, so pick the absolute oldest // version if we're configured to drop items. *page_id = lowest_page; *page_version = lowest_version; *drop_unread = true; return 1; } return 0; } static pthread_t storage_compact_tid; static pthread_mutex_t storage_compact_plock; #define MIN_STORAGE_COMPACT_SLEEP 10000 #define MAX_STORAGE_COMPACT_SLEEP 2000000 struct storage_compact_wrap { obj_io io; pthread_mutex_t lock; // gates the bools. bool done; bool submitted; bool miss; // version flipped out from under us }; static void storage_compact_readback(void *storage, logger *l, bool drop_unread, char *readback_buf, uint32_t page_id, uint64_t page_version, uint64_t read_size) { uint64_t offset = 0; unsigned int rescues = 0; unsigned int lost = 0; unsigned int skipped = 0; while (offset < read_size) { item *hdr_it = NULL; item_hdr *hdr = NULL; item *it = (item *)(readback_buf+offset); unsigned int ntotal; // probably zeroed out junk at the end of the wbuf if (it->nkey == 0) { break; } ntotal = ITEM_ntotal(it); uint32_t hv = (uint32_t)it->time; item_lock(hv); // We don't have a conn and don't need to do most of do_item_get hdr_it = assoc_find(ITEM_key(it), it->nkey, hv); if (hdr_it != NULL) { bool do_write = false; refcount_incr(hdr_it); // Check validity but don't bother removing it. if ((hdr_it->it_flags & ITEM_HDR) && !item_is_flushed(hdr_it) && (hdr_it->exptime == 0 || hdr_it->exptime > current_time)) { hdr = (item_hdr *)ITEM_data(hdr_it); if (hdr->page_id == page_id && hdr->page_version == page_version) { // Item header is still completely valid. extstore_delete(storage, page_id, page_version, 1, ntotal); // drop inactive items. if (drop_unread && GET_LRU(hdr_it->slabs_clsid) == COLD_LRU) { do_write = false; skipped++; } else { do_write = true; } } } if (do_write) { bool do_update = false; int tries; obj_io io; io.len = ntotal; io.mode = OBJ_IO_WRITE; for (tries = 10; tries > 0; tries--) { if (extstore_write_request(storage, PAGE_BUCKET_COMPACT, PAGE_BUCKET_COMPACT, &io) == 0) { memcpy(io.buf, it, io.len); extstore_write(storage, &io); do_update = true; break; } else { usleep(1000); } } if (do_update) { if (it->refcount == 2) { hdr->page_version = io.page_version; hdr->page_id = io.page_id; hdr->offset = io.offset; rescues++; } else { lost++; // TODO: re-alloc and replace header. } } else { lost++; } } do_item_remove(hdr_it); } item_unlock(hv); offset += ntotal; if (read_size - offset < sizeof(struct _stritem)) break; } STATS_LOCK(); stats.extstore_compact_lost += lost; stats.extstore_compact_rescues += rescues; stats.extstore_compact_skipped += skipped; STATS_UNLOCK(); LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_READ_END, NULL, page_id, offset, rescues, lost, skipped); } static void _storage_compact_cb(void *e, obj_io *io, int ret) { struct storage_compact_wrap *wrap = (struct storage_compact_wrap *)io->data; assert(wrap->submitted == true); pthread_mutex_lock(&wrap->lock); if (ret < 1) { wrap->miss = true; } wrap->done = true; pthread_mutex_unlock(&wrap->lock); } // TODO: hoist the storage bits from lru_maintainer_thread in here. // would be nice if they could avoid hammering the same locks though? // I guess it's only COLD. that's probably fine. static void *storage_compact_thread(void *arg) { void *storage = arg; useconds_t to_sleep = MAX_STORAGE_COMPACT_SLEEP; bool compacting = false; uint64_t page_version = 0; uint64_t page_size = 0; uint64_t page_offset = 0; uint32_t page_id = 0; bool drop_unread = false; char *readback_buf = NULL; struct storage_compact_wrap wrap; logger *l = logger_create(); if (l == NULL) { fprintf(stderr, "Failed to allocate logger for storage compaction thread\n"); abort(); } readback_buf = malloc(settings.ext_wbuf_size); if (readback_buf == NULL) { fprintf(stderr, "Failed to allocate readback buffer for storage compaction thread\n"); abort(); } pthread_mutex_init(&wrap.lock, NULL); wrap.done = false; wrap.submitted = false; wrap.io.data = &wrap; wrap.io.buf = (void *)readback_buf; wrap.io.len = settings.ext_wbuf_size; wrap.io.mode = OBJ_IO_READ; wrap.io.cb = _storage_compact_cb; pthread_mutex_lock(&storage_compact_plock); while (1) { pthread_mutex_unlock(&storage_compact_plock); if (to_sleep) { extstore_run_maint(storage); usleep(to_sleep); } pthread_mutex_lock(&storage_compact_plock); if (!compacting && storage_compact_check(storage, l, &page_id, &page_version, &page_size, &drop_unread)) { page_offset = 0; compacting = true; LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_START, NULL, page_id, page_version); } if (compacting) { pthread_mutex_lock(&wrap.lock); if (page_offset < page_size && !wrap.done && !wrap.submitted) { wrap.io.page_version = page_version; wrap.io.page_id = page_id; wrap.io.offset = page_offset; // FIXME: should be smarter about io->next (unlink at use?) wrap.io.next = NULL; wrap.submitted = true; wrap.miss = false; extstore_submit(storage, &wrap.io); } else if (wrap.miss) { LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_ABORT, NULL, page_id); wrap.done = false; wrap.submitted = false; compacting = false; } else if (wrap.submitted && wrap.done) { LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_READ_START, NULL, page_id, page_offset); storage_compact_readback(storage, l, drop_unread, readback_buf, page_id, page_version, settings.ext_wbuf_size); page_offset += settings.ext_wbuf_size; wrap.done = false; wrap.submitted = false; } else if (page_offset >= page_size) { compacting = false; wrap.done = false; wrap.submitted = false; extstore_close_page(storage, page_id, page_version); LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_END, NULL, page_id); } pthread_mutex_unlock(&wrap.lock); if (to_sleep > MIN_STORAGE_COMPACT_SLEEP) to_sleep /= 2; } else { if (to_sleep < MAX_STORAGE_COMPACT_SLEEP) to_sleep += MIN_STORAGE_COMPACT_SLEEP; } } free(readback_buf); return NULL; } // TODO // logger needs logger_destroy() to exist/work before this is safe. /*int stop_storage_compact_thread(void) { int ret; pthread_mutex_lock(&lru_maintainer_lock); do_run_lru_maintainer_thread = 0; pthread_mutex_unlock(&lru_maintainer_lock); if ((ret = pthread_join(lru_maintainer_tid, NULL)) != 0) { fprintf(stderr, "Failed to stop LRU maintainer thread: %s\n", strerror(ret)); return -1; } settings.lru_maintainer_thread = false; return 0; }*/ void storage_compact_pause(void) { pthread_mutex_lock(&storage_compact_plock); } void storage_compact_resume(void) { pthread_mutex_unlock(&storage_compact_plock); } int start_storage_compact_thread(void *arg) { int ret; pthread_mutex_init(&storage_compact_plock, NULL); if ((ret = pthread_create(&storage_compact_tid, NULL, storage_compact_thread, arg)) != 0) { fprintf(stderr, "Can't create storage_compact thread: %s\n", strerror(ret)); return -1; } return 0; } /*** UTILITY ***/ // /path/to/file:100G:bucket1 // FIXME: Modifies argument. copy instead? struct extstore_conf_file *storage_conf_parse(char *arg, unsigned int page_size) { struct extstore_conf_file *cf = NULL; char *b = NULL; char *p = strtok_r(arg, ":", &b); char unit = 0; uint64_t multiplier = 0; int base_size = 0; if (p == NULL) goto error; // First arg is the filepath. cf = calloc(1, sizeof(struct extstore_conf_file)); cf->file = strdup(p); p = strtok_r(NULL, ":", &b); if (p == NULL) { fprintf(stderr, "must supply size to ext_path, ie: ext_path=/f/e:64m (M|G|T|P supported)\n"); goto error; } unit = tolower(p[strlen(p)-1]); p[strlen(p)-1] = '\0'; // sigh. switch (unit) { case 'm': multiplier = 1024 * 1024; break; case 'g': multiplier = 1024 * 1024 * 1024; break; case 't': multiplier = 1024 * 1024; multiplier *= 1024 * 1024; break; case 'p': multiplier = 1024 * 1024; multiplier *= 1024 * 1024 * 1024; break; } base_size = atoi(p); multiplier *= base_size; // page_count is nearest-but-not-larger-than pages * psize cf->page_count = multiplier / page_size; assert(page_size * cf->page_count <= multiplier); // final token would be a default free bucket p = strtok_r(NULL, ",", &b); // TODO: We reuse the original DEFINES for now, // but if lowttl gets split up this needs to be its own set. if (p != NULL) { if (strcmp(p, "compact") == 0) { cf->free_bucket = PAGE_BUCKET_COMPACT; } else if (strcmp(p, "lowttl") == 0) { cf->free_bucket = PAGE_BUCKET_LOWTTL; } else if (strcmp(p, "chunked") == 0) { cf->free_bucket = PAGE_BUCKET_CHUNKED; } else if (strcmp(p, "default") == 0) { cf->free_bucket = PAGE_BUCKET_DEFAULT; } else { fprintf(stderr, "Unknown extstore bucket: %s\n", p); goto error; } } else { // TODO: is this necessary? cf->free_bucket = PAGE_BUCKET_DEFAULT; } // TODO: disabling until compact algorithm is improved. if (cf->free_bucket != PAGE_BUCKET_DEFAULT) { fprintf(stderr, "ext_path only presently supports the default bucket\n"); goto error; } return cf; error: if (cf) { if (cf->file) free(cf->file); free(cf); } return NULL; } #endif memcached-1.5.22/cache.h0000664000175000017500000001053013615376577011671 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #ifndef CACHE_H #define CACHE_H #include #ifdef HAVE_UMEM_H #include #define cache_t umem_cache_t #define cache_alloc(a) umem_cache_alloc(a, UMEM_DEFAULT) #define do_cache_alloc(a) umem_cache_alloc(a, UMEM_DEFAULT) #define cache_free(a, b) umem_cache_free(a, b) #define do_cache_free(a, b) umem_cache_free(a, b) #define cache_create(a,b,c,d,e) umem_cache_create((char*)a, b, c, d, e, NULL, NULL, NULL, 0) #define cache_destroy(a) umem_cache_destroy(a); #else #ifndef NDEBUG /* may be used for debug purposes */ extern int cache_error; #endif /** * Constructor used to initialize allocated objects * * @param obj pointer to the object to initialized. * @param notused1 This parameter is currently not used. * @param notused2 This parameter is currently not used. * @return you should return 0, but currently this is not checked */ typedef int cache_constructor_t(void* obj, void* notused1, int notused2); /** * Destructor used to clean up allocated objects before they are * returned to the operating system. * * @param obj pointer to the object to clean up. * @param notused This parameter is currently not used. * @return you should return 0, but currently this is not checked */ typedef void cache_destructor_t(void* obj, void* notused); /** * Definition of the structure to keep track of the internal details of * the cache allocator. Touching any of these variables results in * undefined behavior. */ typedef struct { /** Mutex to protect access to the structure */ pthread_mutex_t mutex; /** Name of the cache objects in this cache (provided by the caller) */ char *name; /** List of pointers to available buffers in this cache */ void **ptr; /** The size of each element in this cache */ size_t bufsize; /** The capacity of the list of elements */ int freetotal; /** The current number of free elements */ int freecurr; /** The constructor to be called each time we allocate more memory */ cache_constructor_t* constructor; /** The destructor to be called each time before we release memory */ cache_destructor_t* destructor; } cache_t; /** * Create an object cache. * * The object cache will let you allocate objects of the same size. It is fully * MT safe, so you may allocate objects from multiple threads without having to * do any synchronization in the application code. * * @param name the name of the object cache. This name may be used for debug purposes * and may help you track down what kind of object you have problems with * (buffer overruns, leakage etc) * @param bufsize the size of each object in the cache * @param align the alignment requirements of the objects in the cache. * @param constructor the function to be called to initialize memory when we need * to allocate more memory from the os. * @param destructor the function to be called before we release the memory back * to the os. * @return a handle to an object cache if successful, NULL otherwise. */ cache_t* cache_create(const char* name, size_t bufsize, size_t align, cache_constructor_t* constructor, cache_destructor_t* destructor); /** * Destroy an object cache. * * Destroy and invalidate an object cache. You should return all buffers allocated * with cache_alloc by using cache_free before calling this function. Not doing * so results in undefined behavior (the buffers may or may not be invalidated) * * @param handle the handle to the object cache to destroy. */ void cache_destroy(cache_t* handle); /** * Allocate an object from the cache. * * @param handle the handle to the object cache to allocate from * @return a pointer to an initialized object from the cache, or NULL if * the allocation cannot be satisfied. */ void* cache_alloc(cache_t* handle); void* do_cache_alloc(cache_t* handle); /** * Return an object back to the cache. * * The caller should return the object in an initialized state so that * the object may be returned in an expected state from cache_alloc. * * @param handle handle to the object cache to return the object to * @param ptr pointer to the object to return. */ void cache_free(cache_t* handle, void* ptr); void do_cache_free(cache_t* handle, void* ptr); #endif #endif memcached-1.5.22/config.h.in0000664000175000017500000001106513615454045012467 00000000000000/* config.h.in. Generated from configure.ac by autoheader. */ /* Define if building universal (internal helper macro) */ #undef AC_APPLE_UNIVERSAL_BUILD /* Set to nonzero if you want to enable ARMv8 crc32 */ #undef ARM_CRC32 /* Set to nonzero if you want to compile using ASAN */ #undef ASAN /* Set to nonzero if you want to include DTRACE */ #undef ENABLE_DTRACE /* Set to nonzero if you want to include SASL */ #undef ENABLE_SASL /* Set to nonzero if you want to enable a SASL pwdb */ #undef ENABLE_SASL_PWDB /* machine is bigendian */ #undef ENDIAN_BIG /* machine is littleendian */ #undef ENDIAN_LITTLE /* Set to nonzero if you want to enable extstore */ #undef EXTSTORE /* Define to 1 if support accept4 */ #undef HAVE_ACCEPT4 /* Define to 1 if you have the `cap_enter' function. */ #undef HAVE_CAP_ENTER /* Define to 1 if you have the `clock_gettime' function. */ #undef HAVE_CLOCK_GETTIME /* Define this if you have an implementation of drop_privileges() */ #undef HAVE_DROP_PRIVILEGES /* Define this if you have an implementation of drop_worker_privileges() */ #undef HAVE_DROP_WORKER_PRIVILEGES /* GCC 64bit Atomics available */ #undef HAVE_GCC_64ATOMICS /* GCC Atomics available */ #undef HAVE_GCC_ATOMICS /* Define to 1 if support getopt_long */ #undef HAVE_GETOPT_LONG /* Define to 1 if you have the `getpagesizes' function. */ #undef HAVE_GETPAGESIZES /* Have ntohll */ #undef HAVE_HTONLL /* Define to 1 if you have the header file. */ #undef HAVE_INTTYPES_H /* Define to 1 if you have the `memcntl' function. */ #undef HAVE_MEMCNTL /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H /* Define to 1 if you have the `mlockall' function. */ #undef HAVE_MLOCKALL /* Define to 1 if you have the `pledge' function. */ #undef HAVE_PLEDGE /* we have sasl_callback_ft */ #undef HAVE_SASL_CALLBACK_FT /* Set to nonzero if your SASL implementation supports SASL_CB_GETCONF */ #undef HAVE_SASL_CB_GETCONF /* Set to nonzero if your SASL implementation supports SASL_CB_GETCONFPATH */ #undef HAVE_SASL_CB_GETCONFPATH /* Define to 1 if you have the header file. */ #undef HAVE_SASL_SASL_H /* Define to 1 if you have the `setppriv' function. */ #undef HAVE_SETPPRIV /* Define to 1 if you have the `sigignore' function. */ #undef HAVE_SIGIGNORE /* Define to 1 if stdbool.h conforms to C99. */ #undef HAVE_STDBOOL_H /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H /* Define to 1 if you have the header file. */ #undef HAVE_STDLIB_H /* Define to 1 if you have the header file. */ #undef HAVE_STRINGS_H /* Define to 1 if you have the header file. */ #undef HAVE_STRING_H /* Define to 1 if you have the `sysconf' function. */ #undef HAVE_SYSCONF /* Define to 1 if you have the header file. */ #undef HAVE_SYS_STAT_H /* Define to 1 if you have the header file. */ #undef HAVE_SYS_TYPES_H /* Define this if you have umem.h */ #undef HAVE_UMEM_H /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H /* Define to 1 if the system has the type `_Bool'. */ #undef HAVE__BOOL /* Machine need alignment */ #undef NEED_ALIGN /* Name of package */ #undef PACKAGE /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT /* Define to the full name of this package. */ #undef PACKAGE_NAME /* Define to the full name and version of this package. */ #undef PACKAGE_STRING /* Define to the one symbol short name of this package. */ #undef PACKAGE_TARNAME /* Define to the home page for this package. */ #undef PACKAGE_URL /* Define to the version of this package. */ #undef PACKAGE_VERSION /* The size of `void *', as computed by sizeof. */ #undef SIZEOF_VOID_P /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS /* Set to nonzero if you want to enable TLS */ #undef TLS /* Version number of package */ #undef VERSION /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ #if defined AC_APPLE_UNIVERSAL_BUILD # if defined __BIG_ENDIAN__ # define WORDS_BIGENDIAN 1 # endif #else # ifndef WORDS_BIGENDIAN # undef WORDS_BIGENDIAN # endif #endif /* find sigignore on Linux */ #undef _GNU_SOURCE /* Define to empty if `const' does not conform to ANSI C. */ #undef const /* define to int if socklen_t not available */ #undef socklen_t #if HAVE_STDBOOL_H #include #else #define bool char #define false 0 #define true 1 #endif #ifdef HAVE_INTTYPES_H #include #endif memcached-1.5.22/README.md0000664000175000017500000000222713575022500011713 00000000000000# Memcached Memcached is a high performance multithreaded event-based key/value cache store intended to be used in a distributed system. See: https://memcached.org/about A fun story explaining usage: https://memcached.org/tutorial If you're having trouble, try the wiki: https://memcached.org/wiki If you're trying to troubleshoot odd behavior or timeouts, see: https://memcached.org/timeouts https://memcached.org/ is a good resource in general. Please use the mailing list to ask questions, github issues aren't seen by everyone! ## Dependencies * libevent, https://www.monkey.org/~provos/libevent/ (libevent-dev) * libseccomp, (optional, experimental, linux) - enables process restrictions for better security. Tested only on x86_64 architectures. ## Environment Be warned that the -k (mlockall) option to memcached might be dangerous when using a large cache. Just make sure the memcached machines don't swap. memcached does non-blocking network I/O, but not disk. (it should never go to disk, or you've lost the whole point of it) ## Website * https://www.memcached.org ## Contributing See https://github.com/memcached/memcached/wiki/DevelopmentRepos memcached-1.5.22/slabs.h0000664000175000017500000000445213575022500011713 00000000000000/* slabs memory allocation */ #ifndef SLABS_H #define SLABS_H /** Init the subsystem. 1st argument is the limit on no. of bytes to allocate, 0 if no limit. 2nd argument is the growth factor; each slab will use a chunk size equal to the previous slab's chunk size times this factor. 3rd argument specifies if the slab allocator should allocate all memory up front (if true), or allocate memory in chunks as it is needed (if false) */ void slabs_init(const size_t limit, const double factor, const bool prealloc, const uint32_t *slab_sizes, void *mem_base_external, bool reuse_mem); /** Call only during init. Pre-allocates all available memory */ void slabs_prefill_global(void); /** * Given object size, return id to use when allocating/freeing memory for object * 0 means error: can't store such a large object */ unsigned int slabs_clsid(const size_t size); unsigned int slabs_size(const int clsid); /** Allocate object of given length. 0 on error */ /*@null@*/ #define SLABS_ALLOC_NO_NEWPAGE 1 void *slabs_alloc(const size_t size, unsigned int id, unsigned int flags); /** Free previously allocated object */ void slabs_free(void *ptr, size_t size, unsigned int id); /** Adjust global memory limit up or down */ bool slabs_adjust_mem_limit(size_t new_mem_limit); typedef struct { unsigned int chunks_per_page; unsigned int chunk_size; long int free_chunks; long int total_pages; } slab_stats_automove; void fill_slab_stats_automove(slab_stats_automove *am); unsigned int global_page_pool_size(bool *mem_flag); /** Fill buffer with stats */ /*@null@*/ void slabs_stats(ADD_STAT add_stats, void *c); /* Hints as to freespace in slab class */ unsigned int slabs_available_chunks(unsigned int id, bool *mem_flag, unsigned int *chunks_perslab); void slabs_mlock(void); void slabs_munlock(void); int start_slab_maintenance_thread(void); void stop_slab_maintenance_thread(void); enum reassign_result_type { REASSIGN_OK=0, REASSIGN_RUNNING, REASSIGN_BADCLASS, REASSIGN_NOSPARE, REASSIGN_SRC_DST_SAME }; enum reassign_result_type slabs_reassign(int src, int dst); void slabs_rebalancer_pause(void); void slabs_rebalancer_resume(void); #ifdef EXTSTORE void slabs_set_storage(void *arg); #endif /* Fixup for restartable code. */ unsigned int slabs_fixup(char *chunk, const int border); #endif memcached-1.5.22/util.h0000664000175000017500000000243613615376577011611 00000000000000/* fast-enough functions for uriencoding strings. */ void uriencode_init(void); bool uriencode(const char *src, char *dst, const size_t srclen, const size_t dstlen); /* * Wrappers around strtoull/strtoll that are safer and easier to * use. For tests and assumptions, see internal_tests.c. * * str a NULL-terminated base decimal 10 unsigned integer * out out parameter, if conversion succeeded * * returns true if conversion succeeded. */ bool safe_strtoull(const char *str, uint64_t *out); bool safe_strtoull_hex(const char *str, uint64_t *out); bool safe_strtoll(const char *str, int64_t *out); bool safe_strtoul(const char *str, uint32_t *out); bool safe_strtol(const char *str, int32_t *out); bool safe_strtod(const char *str, double *out); bool safe_strcpy(char *dst, const char *src, const size_t dstmax); bool safe_memcmp(const void *a, const void *b, size_t len); #ifndef HAVE_HTONLL extern uint64_t htonll(uint64_t); extern uint64_t ntohll(uint64_t); #endif #ifdef __GCC # define __gcc_attribute__ __attribute__ #else # define __gcc_attribute__(x) #endif /** * Vararg variant of perror that makes for more useful error messages * when reporting with parameters. * * @param fmt a printf format */ void vperror(const char *fmt, ...) __gcc_attribute__ ((format (printf, 1, 2))); memcached-1.5.22/memcached.c0000664000175000017500000124005013615376577012532 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ /* * memcached - memory caching daemon * * https://www.memcached.org/ * * Copyright 2003 Danga Interactive, Inc. All rights reserved. * * Use and distribution licensed under the BSD license. See * the LICENSE file for full text. * * Authors: * Anatoly Vorobey * Brad Fitzpatrick */ #include "memcached.h" #ifdef EXTSTORE #include "storage.h" #endif #include "authfile.h" #include "restart.h" #include #include #include #include #include #include #include #include #include /* some POSIX systems need the following definition * to get mlockall flags out of sys/mman.h. */ #ifndef _P1003_1B_VISIBLE #define _P1003_1B_VISIBLE #endif /* need this to get IOV_MAX on some platforms. */ #ifndef __need_IOV_MAX #define __need_IOV_MAX #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_GETOPT_LONG #include #endif #ifdef TLS #include "tls.h" #endif #if defined(__FreeBSD__) #include #endif /* FreeBSD 4.x doesn't have IOV_MAX exposed. */ #ifndef IOV_MAX #if defined(__FreeBSD__) || defined(__APPLE__) || defined(__GNU__) # define IOV_MAX 1024 /* GNU/Hurd don't set MAXPATHLEN * http://www.gnu.org/software/hurd/hurd/porting/guidelines.html#PATH_MAX_tt_MAX_PATH_tt_MAXPATHL */ #ifndef MAXPATHLEN #define MAXPATHLEN 4096 #endif #endif #endif /* * forward declarations */ static void drive_machine(conn *c); static int new_socket(struct addrinfo *ai); static ssize_t tcp_read(conn *arg, void *buf, size_t count); static ssize_t tcp_sendmsg(conn *arg, struct msghdr *msg, int flags); static ssize_t tcp_write(conn *arg, void *buf, size_t count); enum try_read_result { READ_DATA_RECEIVED, READ_NO_DATA_RECEIVED, READ_ERROR, /** an error occurred (on the socket) (or client closed connection) */ READ_MEMORY_ERROR /** failed to allocate more memory */ }; static int try_read_command_negotiate(conn *c); static int try_read_command_udp(conn *c); static int try_read_command_binary(conn *c); static int try_read_command_ascii(conn *c); static int try_read_command_asciiauth(conn *c); static enum try_read_result try_read_network(conn *c); static enum try_read_result try_read_udp(conn *c); static void conn_set_state(conn *c, enum conn_states state); static int start_conn_timeout_thread(); /* stats */ static void stats_init(void); static void server_stats(ADD_STAT add_stats, conn *c); static void process_stat_settings(ADD_STAT add_stats, void *c); static void conn_to_str(const conn *c, char *addr, char *svr_addr); /** Return a datum for stats in binary protocol */ static bool get_stats(const char *stat_type, int nkey, ADD_STAT add_stats, void *c); /* defaults */ static void settings_init(void); /* event handling, network IO */ static void event_handler(const int fd, const short which, void *arg); static void conn_close(conn *c); static void conn_init(void); static bool update_event(conn *c, const int new_flags); static void complete_nread(conn *c); static void process_command(conn *c, char *command); static void write_and_free(conn *c, char *buf, int bytes); static int ensure_iov_space(conn *c); static int add_iov(conn *c, const void *buf, int len); static int add_chunked_item_iovs(conn *c, item *it, int len); static int add_msghdr(conn *c); static void write_bin_error(conn *c, protocol_binary_response_status err, const char *errstr, int swallow); static void write_bin_miss_response(conn *c, char *key, size_t nkey); #ifdef EXTSTORE static void _get_extstore_cb(void *e, obj_io *io, int ret); static inline int _get_extstore(conn *c, item *it, int iovst, int iovcnt); #endif static void conn_free(conn *c); /** exported globals **/ struct stats stats; struct stats_state stats_state; struct settings settings; time_t process_started; /* when the process was started */ conn **conns; struct slab_rebalance slab_rebal; volatile int slab_rebalance_signal; #ifdef EXTSTORE /* hoping this is temporary; I'd prefer to cut globals, but will complete this * battle another day. */ void *ext_storage = NULL; #endif /** file scope variables **/ static conn *listen_conn = NULL; static int max_fds; static struct event_base *main_base; enum transmit_result { TRANSMIT_COMPLETE, /** All done writing. */ TRANSMIT_INCOMPLETE, /** More data remaining to write. */ TRANSMIT_SOFT_ERROR, /** Can't write any more right now. */ TRANSMIT_HARD_ERROR /** Can't write (c->state is set to conn_closing) */ }; /* Default methods to read from/ write to a socket */ ssize_t tcp_read(conn *c, void *buf, size_t count) { assert (c != NULL); return read(c->sfd, buf, count); } ssize_t tcp_sendmsg(conn *c, struct msghdr *msg, int flags) { assert (c != NULL); return sendmsg(c->sfd, msg, flags); } ssize_t tcp_write(conn *c, void *buf, size_t count) { assert (c != NULL); return write(c->sfd, buf, count); } static enum transmit_result transmit(conn *c); /* This reduces the latency without adding lots of extra wiring to be able to * notify the listener thread of when to listen again. * Also, the clock timer could be broken out into its own thread and we * can block the listener via a condition. */ static volatile bool allow_new_conns = true; static bool stop_main_loop = false; static struct event maxconnsevent; static void maxconns_handler(const int fd, const short which, void *arg) { struct timeval t = {.tv_sec = 0, .tv_usec = 10000}; if (fd == -42 || allow_new_conns == false) { /* reschedule in 10ms if we need to keep polling */ evtimer_set(&maxconnsevent, maxconns_handler, 0); event_base_set(main_base, &maxconnsevent); evtimer_add(&maxconnsevent, &t); } else { evtimer_del(&maxconnsevent); accept_new_conns(true); } } #define REALTIME_MAXDELTA 60*60*24*30 /* * given time value that's either unix time or delta from current unix time, return * unix time. Use the fact that delta can't exceed one month (and real time value can't * be that low). */ static rel_time_t realtime(const time_t exptime) { /* no. of seconds in 30 days - largest possible delta exptime */ if (exptime == 0) return 0; /* 0 means never expire */ if (exptime > REALTIME_MAXDELTA) { /* if item expiration is at/before the server started, give it an expiration time of 1 second after the server started. (because 0 means don't expire). without this, we'd underflow and wrap around to some large value way in the future, effectively making items expiring in the past really expiring never */ if (exptime <= process_started) return (rel_time_t)1; return (rel_time_t)(exptime - process_started); } else { return (rel_time_t)(exptime + current_time); } } static void stats_init(void) { memset(&stats, 0, sizeof(struct stats)); memset(&stats_state, 0, sizeof(struct stats_state)); stats_state.accepting_conns = true; /* assuming we start in this state. */ /* make the time we started always be 2 seconds before we really did, so time(0) - time.started is never zero. if so, things like 'settings.oldest_live' which act as booleans as well as values are now false in boolean context... */ process_started = time(0) - ITEM_UPDATE_INTERVAL - 2; stats_prefix_init(settings.prefix_delimiter); } static void stats_reset(void) { STATS_LOCK(); memset(&stats, 0, sizeof(struct stats)); stats_prefix_clear(); STATS_UNLOCK(); threadlocal_stats_reset(); item_stats_reset(); } static void settings_init(void) { settings.use_cas = true; settings.access = 0700; settings.port = 11211; settings.udpport = 0; #ifdef TLS settings.ssl_enabled = false; settings.ssl_ctx = NULL; settings.ssl_chain_cert = NULL; settings.ssl_key = NULL; settings.ssl_verify_mode = SSL_VERIFY_NONE; settings.ssl_keyformat = SSL_FILETYPE_PEM; settings.ssl_ciphers = NULL; settings.ssl_ca_cert = NULL; settings.ssl_last_cert_refresh_time = current_time; settings.ssl_wbuf_size = 16 * 1024; // default is 16KB (SSL max frame size is 17KB) #endif /* By default this string should be NULL for getaddrinfo() */ settings.inter = NULL; settings.maxbytes = 64 * 1024 * 1024; /* default is 64MB */ settings.maxconns = 1024; /* to limit connections-related memory to about 5MB */ settings.verbose = 0; settings.oldest_live = 0; settings.oldest_cas = 0; /* supplements accuracy of oldest_live */ settings.evict_to_free = 1; /* push old items out of cache when memory runs out */ settings.socketpath = NULL; /* by default, not using a unix socket */ settings.auth_file = NULL; /* by default, not using ASCII authentication tokens */ settings.factor = 1.25; settings.chunk_size = 48; /* space for a modest key and value */ settings.num_threads = 4; /* N workers */ settings.num_threads_per_udp = 0; settings.prefix_delimiter = ':'; settings.detail_enabled = 0; settings.reqs_per_event = 20; settings.backlog = 1024; settings.binding_protocol = negotiating_prot; settings.item_size_max = 1024 * 1024; /* The famous 1MB upper limit. */ settings.slab_page_size = 1024 * 1024; /* chunks are split from 1MB pages. */ settings.slab_chunk_size_max = settings.slab_page_size / 2; settings.sasl = false; settings.maxconns_fast = true; settings.lru_crawler = false; settings.lru_crawler_sleep = 100; settings.lru_crawler_tocrawl = 0; settings.lru_maintainer_thread = false; settings.lru_segmented = true; settings.hot_lru_pct = 20; settings.warm_lru_pct = 40; settings.hot_max_factor = 0.2; settings.warm_max_factor = 2.0; settings.temp_lru = false; settings.temporary_ttl = 61; settings.idle_timeout = 0; /* disabled */ settings.hashpower_init = 0; settings.slab_reassign = true; settings.slab_automove = 1; settings.slab_automove_ratio = 0.8; settings.slab_automove_window = 30; settings.shutdown_command = false; settings.tail_repair_time = TAIL_REPAIR_TIME_DEFAULT; settings.flush_enabled = true; settings.dump_enabled = true; settings.crawls_persleep = 1000; settings.logger_watcher_buf_size = LOGGER_WATCHER_BUF_SIZE; settings.logger_buf_size = LOGGER_BUF_SIZE; settings.drop_privileges = false; settings.watch_enabled = true; #ifdef MEMCACHED_DEBUG settings.relaxed_privileges = false; #endif } /* * Adds a message header to a connection. * * Returns 0 on success, -1 on out-of-memory. */ static int add_msghdr(conn *c) { struct msghdr *msg; assert(c != NULL); if (c->msgsize == c->msgused) { msg = realloc(c->msglist, c->msgsize * 2 * sizeof(struct msghdr)); if (! msg) { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); return -1; } c->msglist = msg; c->msgsize *= 2; } msg = c->msglist + c->msgused; /* this wipes msg_iovlen, msg_control, msg_controllen, and msg_flags, the last 3 of which aren't defined on solaris: */ memset(msg, 0, sizeof(struct msghdr)); msg->msg_iov = &c->iov[c->iovused]; if (IS_UDP(c->transport) && c->request_addr_size > 0) { msg->msg_name = &c->request_addr; msg->msg_namelen = c->request_addr_size; } c->msgbytes = 0; c->msgused++; if (IS_UDP(c->transport)) { /* Leave room for the UDP header, which we'll fill in later. */ return add_iov(c, NULL, UDP_HEADER_SIZE); } return 0; } extern pthread_mutex_t conn_lock; /* Connection timeout thread bits */ static pthread_t conn_timeout_tid; static int do_run_conn_timeout_thread; #define CONNS_PER_SLICE 100 #define TIMEOUT_MSG_SIZE (1 + sizeof(int)) static void *conn_timeout_thread(void *arg) { int i; conn *c; char buf[TIMEOUT_MSG_SIZE]; rel_time_t oldest_last_cmd; int sleep_time; useconds_t timeslice = 1000000 / (max_fds / CONNS_PER_SLICE); while(do_run_conn_timeout_thread) { if (settings.verbose > 2) fprintf(stderr, "idle timeout thread at top of connection list\n"); oldest_last_cmd = current_time; for (i = 0; i < max_fds; i++) { if ((i % CONNS_PER_SLICE) == 0) { if (settings.verbose > 2) fprintf(stderr, "idle timeout thread sleeping for %ulus\n", (unsigned int)timeslice); usleep(timeslice); } if (!conns[i]) continue; c = conns[i]; if (!IS_TCP(c->transport)) continue; if (c->state != conn_new_cmd && c->state != conn_read) continue; if ((current_time - c->last_cmd_time) > settings.idle_timeout) { buf[0] = 't'; memcpy(&buf[1], &i, sizeof(int)); if (write(c->thread->notify_send_fd, buf, TIMEOUT_MSG_SIZE) != TIMEOUT_MSG_SIZE) perror("Failed to write timeout to notify pipe"); } else { if (c->last_cmd_time < oldest_last_cmd) oldest_last_cmd = c->last_cmd_time; } } /* This is the soonest we could have another connection time out */ sleep_time = settings.idle_timeout - (current_time - oldest_last_cmd) + 1; if (sleep_time <= 0) sleep_time = 1; if (settings.verbose > 2) fprintf(stderr, "idle timeout thread finished pass, sleeping for %ds\n", sleep_time); usleep((useconds_t) sleep_time * 1000000); } return NULL; } static int start_conn_timeout_thread() { int ret; if (settings.idle_timeout == 0) return -1; do_run_conn_timeout_thread = 1; if ((ret = pthread_create(&conn_timeout_tid, NULL, conn_timeout_thread, NULL)) != 0) { fprintf(stderr, "Can't create idle connection timeout thread: %s\n", strerror(ret)); return -1; } return 0; } int stop_conn_timeout_thread(void) { if (!do_run_conn_timeout_thread) return -1; do_run_conn_timeout_thread = 0; pthread_join(conn_timeout_tid, NULL); return 0; } /* * Initializes the connections array. We don't actually allocate connection * structures until they're needed, so as to avoid wasting memory when the * maximum connection count is much higher than the actual number of * connections. * * This does end up wasting a few pointers' worth of memory for FDs that are * used for things other than connections, but that's worth it in exchange for * being able to directly index the conns array by FD. */ static void conn_init(void) { /* We're unlikely to see an FD much higher than maxconns. */ int next_fd = dup(1); if (next_fd < 0) { perror("Failed to duplicate file descriptor\n"); exit(1); } int headroom = 10; /* account for extra unexpected open FDs */ struct rlimit rl; max_fds = settings.maxconns + headroom + next_fd; /* But if possible, get the actual highest FD we can possibly ever see. */ if (getrlimit(RLIMIT_NOFILE, &rl) == 0) { max_fds = rl.rlim_max; } else { fprintf(stderr, "Failed to query maximum file descriptor; " "falling back to maxconns\n"); } close(next_fd); if ((conns = calloc(max_fds, sizeof(conn *))) == NULL) { fprintf(stderr, "Failed to allocate connection structures\n"); /* This is unrecoverable so bail out early. */ exit(1); } } static const char *prot_text(enum protocol prot) { char *rv = "unknown"; switch(prot) { case ascii_prot: rv = "ascii"; break; case binary_prot: rv = "binary"; break; case negotiating_prot: rv = "auto-negotiate"; break; } return rv; } void conn_close_idle(conn *c) { if (settings.idle_timeout > 0 && (current_time - c->last_cmd_time) > settings.idle_timeout) { if (c->state != conn_new_cmd && c->state != conn_read) { if (settings.verbose > 1) fprintf(stderr, "fd %d wants to timeout, but isn't in read state", c->sfd); return; } if (settings.verbose > 1) fprintf(stderr, "Closing idle fd %d\n", c->sfd); c->thread->stats.idle_kicks++; conn_set_state(c, conn_closing); drive_machine(c); } } /* bring conn back from a sidethread. could have had its event base moved. */ void conn_worker_readd(conn *c) { c->ev_flags = EV_READ | EV_PERSIST; event_set(&c->event, c->sfd, c->ev_flags, event_handler, (void *)c); event_base_set(c->thread->base, &c->event); c->state = conn_new_cmd; // TODO: call conn_cleanup/fail/etc if (event_add(&c->event, 0) == -1) { perror("event_add"); } #ifdef EXTSTORE // If we had IO objects, process if (c->io_wraplist) { //assert(c->io_wrapleft == 0); // assert no more to process conn_set_state(c, conn_mwrite); drive_machine(c); } #endif } conn *conn_new(const int sfd, enum conn_states init_state, const int event_flags, const int read_buffer_size, enum network_transport transport, struct event_base *base, void *ssl) { conn *c; assert(sfd >= 0 && sfd < max_fds); c = conns[sfd]; if (NULL == c) { if (!(c = (conn *)calloc(1, sizeof(conn)))) { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); fprintf(stderr, "Failed to allocate connection object\n"); return NULL; } MEMCACHED_CONN_CREATE(c); c->read = NULL; c->sendmsg = NULL; c->write = NULL; c->rbuf = c->wbuf = 0; c->ilist = 0; c->suffixlist = 0; c->iov = 0; c->msglist = 0; c->hdrbuf = 0; c->rsize = read_buffer_size; c->wsize = DATA_BUFFER_SIZE; c->isize = ITEM_LIST_INITIAL; c->suffixsize = SUFFIX_LIST_INITIAL; c->iovsize = IOV_LIST_INITIAL; c->msgsize = MSG_LIST_INITIAL; c->hdrsize = 0; c->rbuf = (char *)malloc((size_t)c->rsize); c->wbuf = (char *)malloc((size_t)c->wsize); c->ilist = (item **)malloc(sizeof(item *) * c->isize); c->suffixlist = (char **)malloc(sizeof(char *) * c->suffixsize); c->iov = (struct iovec *)malloc(sizeof(struct iovec) * c->iovsize); c->msglist = (struct msghdr *)malloc(sizeof(struct msghdr) * c->msgsize); if (c->rbuf == 0 || c->wbuf == 0 || c->ilist == 0 || c->iov == 0 || c->msglist == 0 || c->suffixlist == 0) { conn_free(c); STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); fprintf(stderr, "Failed to allocate buffers for connection\n"); return NULL; } STATS_LOCK(); stats_state.conn_structs++; STATS_UNLOCK(); c->sfd = sfd; conns[sfd] = c; } c->transport = transport; c->protocol = settings.binding_protocol; /* unix socket mode doesn't need this, so zeroed out. but why * is this done for every command? presumably for UDP * mode. */ if (!settings.socketpath) { c->request_addr_size = sizeof(c->request_addr); } else { c->request_addr_size = 0; } if (transport == tcp_transport && init_state == conn_new_cmd) { if (getpeername(sfd, (struct sockaddr *) &c->request_addr, &c->request_addr_size)) { perror("getpeername"); memset(&c->request_addr, 0, sizeof(c->request_addr)); } } if (settings.verbose > 1) { if (init_state == conn_listening) { fprintf(stderr, "<%d server listening (%s)\n", sfd, prot_text(c->protocol)); } else if (IS_UDP(transport)) { fprintf(stderr, "<%d server listening (udp)\n", sfd); } else if (c->protocol == negotiating_prot) { fprintf(stderr, "<%d new auto-negotiating client connection\n", sfd); } else if (c->protocol == ascii_prot) { fprintf(stderr, "<%d new ascii client connection.\n", sfd); } else if (c->protocol == binary_prot) { fprintf(stderr, "<%d new binary client connection.\n", sfd); } else { fprintf(stderr, "<%d new unknown (%d) client connection\n", sfd, c->protocol); assert(false); } } #ifdef TLS c->ssl = NULL; c->ssl_wbuf = NULL; c->ssl_enabled = false; #endif c->state = init_state; c->rlbytes = 0; c->cmd = -1; c->rbytes = c->wbytes = 0; c->wcurr = c->wbuf; c->rcurr = c->rbuf; c->ritem = 0; c->icurr = c->ilist; c->suffixcurr = c->suffixlist; c->ileft = 0; c->suffixleft = 0; c->iovused = 0; c->msgcurr = 0; c->msgused = 0; c->sasl_started = false; c->set_stale = false; c->mset_res = false; c->last_cmd_time = current_time; /* initialize for idle kicker */ #ifdef EXTSTORE c->io_wraplist = NULL; c->io_wrapleft = 0; #endif c->write_and_go = init_state; c->write_and_free = 0; c->item = 0; c->noreply = false; #ifdef TLS if (ssl) { c->ssl = (SSL*)ssl; c->read = ssl_read; c->sendmsg = ssl_sendmsg; c->write = ssl_write; c->ssl_enabled = true; SSL_set_info_callback(c->ssl, ssl_callback); } else #else // This must be NULL if TLS is not enabled. assert(ssl == NULL); #endif { c->read = tcp_read; c->sendmsg = tcp_sendmsg; c->write = tcp_write; } if (IS_UDP(transport)) { c->try_read_command = try_read_command_udp; } else { switch (c->protocol) { case ascii_prot: if (settings.auth_file == NULL) { c->authenticated = true; c->try_read_command = try_read_command_ascii; } else { c->authenticated = false; c->try_read_command = try_read_command_asciiauth; } break; case binary_prot: // binprot handles its own authentication via SASL parsing. c->authenticated = false; c->try_read_command = try_read_command_binary; break; case negotiating_prot: c->try_read_command = try_read_command_negotiate; break; } } event_set(&c->event, sfd, event_flags, event_handler, (void *)c); event_base_set(base, &c->event); c->ev_flags = event_flags; if (event_add(&c->event, 0) == -1) { perror("event_add"); return NULL; } STATS_LOCK(); stats_state.curr_conns++; stats.total_conns++; STATS_UNLOCK(); MEMCACHED_CONN_ALLOCATE(c->sfd); return c; } #ifdef EXTSTORE static void recache_or_free(conn *c, io_wrap *wrap) { item *it; it = (item *)wrap->io.buf; bool do_free = true; if (wrap->active) { // If request never dispatched, free the read buffer but leave the // item header alone. do_free = false; size_t ntotal = ITEM_ntotal(wrap->hdr_it); slabs_free(it, ntotal, slabs_clsid(ntotal)); c->io_wrapleft--; assert(c->io_wrapleft >= 0); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.get_aborted_extstore++; pthread_mutex_unlock(&c->thread->stats.mutex); } else if (wrap->miss) { // If request was ultimately a miss, unlink the header. do_free = false; size_t ntotal = ITEM_ntotal(wrap->hdr_it); item_unlink(wrap->hdr_it); slabs_free(it, ntotal, slabs_clsid(ntotal)); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.miss_from_extstore++; if (wrap->badcrc) c->thread->stats.badcrc_from_extstore++; pthread_mutex_unlock(&c->thread->stats.mutex); } else if (settings.ext_recache_rate) { // hashvalue is cuddled during store uint32_t hv = (uint32_t)it->time; // opt to throw away rather than wait on a lock. void *hold_lock = item_trylock(hv); if (hold_lock != NULL) { item *h_it = wrap->hdr_it; uint8_t flags = ITEM_LINKED|ITEM_FETCHED|ITEM_ACTIVE; // Item must be recently hit at least twice to recache. if (((h_it->it_flags & flags) == flags) && h_it->time > current_time - ITEM_UPDATE_INTERVAL && c->recache_counter++ % settings.ext_recache_rate == 0) { do_free = false; // In case it's been updated. it->exptime = h_it->exptime; it->it_flags &= ~ITEM_LINKED; it->refcount = 0; it->h_next = NULL; // might not be necessary. STORAGE_delete(c->thread->storage, h_it); item_replace(h_it, it, hv); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.recache_from_extstore++; pthread_mutex_unlock(&c->thread->stats.mutex); } } if (hold_lock) item_trylock_unlock(hold_lock); } if (do_free) slabs_free(it, ITEM_ntotal(it), ITEM_clsid(it)); wrap->io.buf = NULL; // sanity. wrap->io.next = NULL; wrap->next = NULL; wrap->active = false; // TODO: reuse lock and/or hv. item_remove(wrap->hdr_it); } #endif static void conn_release_items(conn *c) { assert(c != NULL); if (c->item) { item_remove(c->item); c->item = 0; } while (c->ileft > 0) { item *it = *(c->icurr); assert((it->it_flags & ITEM_SLABBED) == 0); item_remove(it); c->icurr++; c->ileft--; } if (c->suffixleft != 0) { for (; c->suffixleft > 0; c->suffixleft--, c->suffixcurr++) { do_cache_free(c->thread->suffix_cache, *(c->suffixcurr)); } } #ifdef EXTSTORE if (c->io_wraplist) { io_wrap *tmp = c->io_wraplist; while (tmp) { io_wrap *next = tmp->next; recache_or_free(c, tmp); do_cache_free(c->thread->io_cache, tmp); // lockless tmp = next; } c->io_wraplist = NULL; } #endif c->icurr = c->ilist; c->suffixcurr = c->suffixlist; } static void conn_cleanup(conn *c) { assert(c != NULL); conn_release_items(c); if (c->write_and_free) { free(c->write_and_free); c->write_and_free = 0; } if (c->sasl_conn) { assert(settings.sasl); sasl_dispose(&c->sasl_conn); c->sasl_conn = NULL; } if (IS_UDP(c->transport)) { conn_set_state(c, conn_read); } } /* * Frees a connection. */ void conn_free(conn *c) { if (c) { assert(c != NULL); assert(c->sfd >= 0 && c->sfd < max_fds); MEMCACHED_CONN_DESTROY(c); conns[c->sfd] = NULL; if (c->hdrbuf) free(c->hdrbuf); if (c->msglist) free(c->msglist); if (c->rbuf) free(c->rbuf); if (c->wbuf) free(c->wbuf); if (c->ilist) free(c->ilist); if (c->suffixlist) free(c->suffixlist); if (c->iov) free(c->iov); #ifdef TLS if (c->ssl_wbuf) c->ssl_wbuf = NULL; #endif free(c); } } static void conn_close(conn *c) { assert(c != NULL); /* delete the event, the socket and the conn */ event_del(&c->event); if (settings.verbose > 1) fprintf(stderr, "<%d connection closed.\n", c->sfd); conn_cleanup(c); MEMCACHED_CONN_RELEASE(c->sfd); conn_set_state(c, conn_closed); #ifdef TLS if (c->ssl) { SSL_shutdown(c->ssl); SSL_free(c->ssl); } #endif close(c->sfd); pthread_mutex_lock(&conn_lock); allow_new_conns = true; pthread_mutex_unlock(&conn_lock); STATS_LOCK(); stats_state.curr_conns--; STATS_UNLOCK(); return; } /* * Shrinks a connection's buffers if they're too big. This prevents * periodic large "get" requests from permanently chewing lots of server * memory. * * This should only be called in between requests since it can wipe output * buffers! */ static void conn_shrink(conn *c) { assert(c != NULL); if (IS_UDP(c->transport)) return; if (c->rsize > READ_BUFFER_HIGHWAT && c->rbytes < DATA_BUFFER_SIZE) { char *newbuf; if (c->rcurr != c->rbuf) memmove(c->rbuf, c->rcurr, (size_t)c->rbytes); newbuf = (char *)realloc((void *)c->rbuf, DATA_BUFFER_SIZE); if (newbuf) { c->rbuf = newbuf; c->rsize = DATA_BUFFER_SIZE; } /* TODO check other branch... */ c->rcurr = c->rbuf; } if (c->isize > ITEM_LIST_HIGHWAT) { item **newbuf = (item**) realloc((void *)c->ilist, ITEM_LIST_INITIAL * sizeof(c->ilist[0])); if (newbuf) { c->ilist = newbuf; c->isize = ITEM_LIST_INITIAL; } /* TODO check error condition? */ } if (c->msgsize > MSG_LIST_HIGHWAT) { struct msghdr *newbuf = (struct msghdr *) realloc((void *)c->msglist, MSG_LIST_INITIAL * sizeof(c->msglist[0])); if (newbuf) { c->msglist = newbuf; c->msgsize = MSG_LIST_INITIAL; } /* TODO check error condition? */ } if (c->iovsize > IOV_LIST_HIGHWAT) { struct iovec *newbuf = (struct iovec *) realloc((void *)c->iov, IOV_LIST_INITIAL * sizeof(c->iov[0])); if (newbuf) { c->iov = newbuf; c->iovsize = IOV_LIST_INITIAL; } /* TODO check return value */ } } /** * Convert a state name to a human readable form. */ static const char *state_text(enum conn_states state) { const char* const statenames[] = { "conn_listening", "conn_new_cmd", "conn_waiting", "conn_read", "conn_parse_cmd", "conn_write", "conn_nread", "conn_swallow", "conn_closing", "conn_mwrite", "conn_closed", "conn_watch" }; return statenames[state]; } /* * Sets a connection's current state in the state machine. Any special * processing that needs to happen on certain state transitions can * happen here. */ static void conn_set_state(conn *c, enum conn_states state) { assert(c != NULL); assert(state >= conn_listening && state < conn_max_state); if (state != c->state) { if (settings.verbose > 2) { fprintf(stderr, "%d: going from %s to %s\n", c->sfd, state_text(c->state), state_text(state)); } if (state == conn_write || state == conn_mwrite) { MEMCACHED_PROCESS_COMMAND_END(c->sfd, c->wbuf, c->wbytes); } c->state = state; } } /* * Ensures that there is room for another struct iovec in a connection's * iov list. * * Returns 0 on success, -1 on out-of-memory. */ static int ensure_iov_space(conn *c) { assert(c != NULL); if (c->iovused >= c->iovsize) { int i, iovnum; struct iovec *new_iov = (struct iovec *)realloc(c->iov, (c->iovsize * 2) * sizeof(struct iovec)); if (! new_iov) { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); return -1; } c->iov = new_iov; c->iovsize *= 2; /* Point all the msghdr structures at the new list. */ for (i = 0, iovnum = 0; i < c->msgused; i++) { c->msglist[i].msg_iov = &c->iov[iovnum]; iovnum += c->msglist[i].msg_iovlen; } } return 0; } /* * Adds data to the list of pending data that will be written out to a * connection. * * Returns 0 on success, -1 on out-of-memory. * Note: This is a hot path for at least ASCII protocol. While there is * redundant code in splitting TCP/UDP handling, any reduction in steps has a * large impact for TCP connections. */ static int add_iov(conn *c, const void *buf, int len) { struct msghdr *m; int leftover; assert(c != NULL); if (IS_UDP(c->transport)) { do { m = &c->msglist[c->msgused - 1]; /* * Limit UDP packets to UDP_MAX_PAYLOAD_SIZE bytes. */ /* We may need to start a new msghdr if this one is full. */ if (m->msg_iovlen == IOV_MAX || (c->msgbytes >= UDP_MAX_PAYLOAD_SIZE)) { add_msghdr(c); m = &c->msglist[c->msgused - 1]; } if (ensure_iov_space(c) != 0) return -1; /* If the fragment is too big to fit in the datagram, split it up */ if (len + c->msgbytes > UDP_MAX_PAYLOAD_SIZE) { leftover = len + c->msgbytes - UDP_MAX_PAYLOAD_SIZE; len -= leftover; } else { leftover = 0; } m = &c->msglist[c->msgused - 1]; m->msg_iov[m->msg_iovlen].iov_base = (void *)buf; m->msg_iov[m->msg_iovlen].iov_len = len; c->msgbytes += len; c->iovused++; m->msg_iovlen++; buf = ((char *)buf) + len; len = leftover; } while (leftover > 0); } else { /* Optimized path for TCP connections */ m = &c->msglist[c->msgused - 1]; if (m->msg_iovlen == IOV_MAX) { add_msghdr(c); m = &c->msglist[c->msgused - 1]; } if (ensure_iov_space(c) != 0) return -1; m->msg_iov[m->msg_iovlen].iov_base = (void *)buf; m->msg_iov[m->msg_iovlen].iov_len = len; c->msgbytes += len; c->iovused++; m->msg_iovlen++; } return 0; } static int add_chunked_item_iovs(conn *c, item *it, int len) { assert(it->it_flags & ITEM_CHUNKED); item_chunk *ch = (item_chunk *) ITEM_schunk(it); while (ch) { int todo = (len > ch->used) ? ch->used : len; if (add_iov(c, ch->data, todo) != 0) { return -1; } ch = ch->next; len -= todo; } return 0; } /* * Constructs a set of UDP headers and attaches them to the outgoing messages. */ static int build_udp_headers(conn *c) { int i; unsigned char *hdr; assert(c != NULL); if (c->msgused > c->hdrsize) { void *new_hdrbuf; if (c->hdrbuf) { new_hdrbuf = realloc(c->hdrbuf, c->msgused * 2 * UDP_HEADER_SIZE); } else { new_hdrbuf = malloc(c->msgused * 2 * UDP_HEADER_SIZE); } if (! new_hdrbuf) { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); return -1; } c->hdrbuf = (unsigned char *)new_hdrbuf; c->hdrsize = c->msgused * 2; } hdr = c->hdrbuf; for (i = 0; i < c->msgused; i++) { c->msglist[i].msg_iov[0].iov_base = (void*)hdr; c->msglist[i].msg_iov[0].iov_len = UDP_HEADER_SIZE; *hdr++ = c->request_id / 256; *hdr++ = c->request_id % 256; *hdr++ = i / 256; *hdr++ = i % 256; *hdr++ = c->msgused / 256; *hdr++ = c->msgused % 256; *hdr++ = 0; *hdr++ = 0; assert((void *) hdr == (caddr_t)c->msglist[i].msg_iov[0].iov_base + UDP_HEADER_SIZE); } return 0; } static void out_string(conn *c, const char *str) { size_t len; assert(c != NULL); if (c->noreply) { if (settings.verbose > 1) fprintf(stderr, ">%d NOREPLY %s\n", c->sfd, str); c->noreply = false; conn_set_state(c, conn_new_cmd); return; } if (settings.verbose > 1) fprintf(stderr, ">%d %s\n", c->sfd, str); /* Nuke a partial output... */ c->msgcurr = 0; c->msgused = 0; c->iovused = 0; add_msghdr(c); len = strlen(str); if ((len + 2) > c->wsize) { /* ought to be always enough. just fail for simplicity */ str = "SERVER_ERROR output line too long"; len = strlen(str); } memcpy(c->wbuf, str, len); memcpy(c->wbuf + len, "\r\n", 2); c->wbytes = len + 2; c->wcurr = c->wbuf; conn_set_state(c, conn_write); c->write_and_go = conn_new_cmd; return; } // For metaget-style ASCII commands. Ignores noreply, ensuring clients see // protocol level errors. static void out_errstring(conn *c, const char *str) { c->noreply = false; out_string(c, str); } #define ALLOW_NOREPLY true #define DISABLE_NOREPLY false // Works from a string already written into c->wbuf. // Called from meta commands. process_command() already zeroes out an output, // and this must be used for nominal use cases. static void out_mstring(conn *c, bool use_noreply) { assert(c != NULL); assert(c->wbytes != 0); if (c->noreply && use_noreply) { c->noreply = false; conn_set_state(c, conn_new_cmd); return; } memcpy(c->wbuf + c->wbytes, "\r\n", 2); c->wbytes += 2; c->wcurr = c->wbuf; conn_set_state(c, conn_write); c->write_and_go = conn_new_cmd; return; } /* * Outputs a protocol-specific "out of memory" error. For ASCII clients, * this is equivalent to out_string(). */ static void out_of_memory(conn *c, char *ascii_error) { const static char error_prefix[] = "SERVER_ERROR "; const static int error_prefix_len = sizeof(error_prefix) - 1; if (c->protocol == binary_prot) { /* Strip off the generic error prefix; it's irrelevant in binary */ if (!strncmp(ascii_error, error_prefix, error_prefix_len)) { ascii_error += error_prefix_len; } write_bin_error(c, PROTOCOL_BINARY_RESPONSE_ENOMEM, ascii_error, 0); } else { out_string(c, ascii_error); } } /* * we get here after reading the value in set/add/replace commands. The command * has been stored in c->cmd, and the item is ready in c->item. */ static void complete_nread_ascii(conn *c) { assert(c != NULL); item *it = c->item; int comm = c->cmd; enum store_item_type ret; bool is_valid = false; pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.slab_stats[ITEM_clsid(it)].set_cmds++; pthread_mutex_unlock(&c->thread->stats.mutex); if ((it->it_flags & ITEM_CHUNKED) == 0) { if (strncmp(ITEM_data(it) + it->nbytes - 2, "\r\n", 2) == 0) { is_valid = true; } } else { char buf[2]; /* should point to the final item chunk */ item_chunk *ch = (item_chunk *) c->ritem; assert(ch->used != 0); /* :( We need to look at the last two bytes. This could span two * chunks. */ if (ch->used > 1) { buf[0] = ch->data[ch->used - 2]; buf[1] = ch->data[ch->used - 1]; } else { assert(ch->prev); assert(ch->used == 1); buf[0] = ch->prev->data[ch->prev->used - 1]; buf[1] = ch->data[ch->used - 1]; } if (strncmp(buf, "\r\n", 2) == 0) { is_valid = true; } else { assert(1 == 0); } } if (!is_valid) { // metaset mode always returns errors. if (c->mset_res) { c->noreply = false; } out_string(c, "CLIENT_ERROR bad data chunk"); } else { ret = store_item(it, comm, c); #ifdef ENABLE_DTRACE uint64_t cas = ITEM_get_cas(it); switch (c->cmd) { case NREAD_ADD: MEMCACHED_COMMAND_ADD(c->sfd, ITEM_key(it), it->nkey, (ret == 1) ? it->nbytes : -1, cas); break; case NREAD_REPLACE: MEMCACHED_COMMAND_REPLACE(c->sfd, ITEM_key(it), it->nkey, (ret == 1) ? it->nbytes : -1, cas); break; case NREAD_APPEND: MEMCACHED_COMMAND_APPEND(c->sfd, ITEM_key(it), it->nkey, (ret == 1) ? it->nbytes : -1, cas); break; case NREAD_PREPEND: MEMCACHED_COMMAND_PREPEND(c->sfd, ITEM_key(it), it->nkey, (ret == 1) ? it->nbytes : -1, cas); break; case NREAD_SET: MEMCACHED_COMMAND_SET(c->sfd, ITEM_key(it), it->nkey, (ret == 1) ? it->nbytes : -1, cas); break; case NREAD_CAS: MEMCACHED_COMMAND_CAS(c->sfd, ITEM_key(it), it->nkey, it->nbytes, cas); break; } #endif if (c->mset_res) { switch (ret) { case STORED: memcpy(c->wbuf, "ST ", 3); out_mstring(c, ALLOW_NOREPLY); break; case EXISTS: memcpy(c->wbuf, "EX ", 3); out_mstring(c, DISABLE_NOREPLY); break; case NOT_FOUND: memcpy(c->wbuf, "NF ", 3); out_mstring(c, DISABLE_NOREPLY); break; case NOT_STORED: memcpy(c->wbuf, "NS ", 3); out_mstring(c, DISABLE_NOREPLY); break; default: c->noreply = false; out_string(c, "SERVER_ERROR Unhandled storage type."); } } else { switch (ret) { case STORED: out_string(c, "STORED"); break; case EXISTS: out_string(c, "EXISTS"); break; case NOT_FOUND: out_string(c, "NOT_FOUND"); break; case NOT_STORED: out_string(c, "NOT_STORED"); break; default: out_string(c, "SERVER_ERROR Unhandled storage type."); } } } c->set_stale = false; /* force flag to be off just in case */ c->mset_res = false; item_remove(c->item); /* release the c->item reference */ c->item = 0; } /** * get a pointer to the start of the request struct for the current command */ static void* binary_get_request(conn *c) { char *ret = c->rcurr; ret -= (sizeof(c->binary_header) + c->binary_header.request.keylen + c->binary_header.request.extlen); assert(ret >= c->rbuf); return ret; } /** * get a pointer to the key in this request */ static char* binary_get_key(conn *c) { return c->rcurr - (c->binary_header.request.keylen); } static void add_bin_header(conn *c, uint16_t err, uint8_t hdr_len, uint16_t key_len, uint32_t body_len) { protocol_binary_response_header* header; assert(c); c->msgcurr = 0; c->msgused = 0; c->iovused = 0; if (add_msghdr(c) != 0) { /* This should never run out of memory because iov and msg lists * have minimum sizes big enough to hold an error response. */ out_of_memory(c, "SERVER_ERROR out of memory adding binary header"); return; } header = (protocol_binary_response_header *)c->wbuf; header->response.magic = (uint8_t)PROTOCOL_BINARY_RES; header->response.opcode = c->binary_header.request.opcode; header->response.keylen = (uint16_t)htons(key_len); header->response.extlen = (uint8_t)hdr_len; header->response.datatype = (uint8_t)PROTOCOL_BINARY_RAW_BYTES; header->response.status = (uint16_t)htons(err); header->response.bodylen = htonl(body_len); header->response.opaque = c->opaque; header->response.cas = htonll(c->cas); if (settings.verbose > 1) { int ii; fprintf(stderr, ">%d Writing bin response:", c->sfd); for (ii = 0; ii < sizeof(header->bytes); ++ii) { if (ii % 4 == 0) { fprintf(stderr, "\n>%d ", c->sfd); } fprintf(stderr, " 0x%02x", header->bytes[ii]); } fprintf(stderr, "\n"); } add_iov(c, c->wbuf, sizeof(header->response)); } /** * Writes a binary error response. If errstr is supplied, it is used as the * error text; otherwise a generic description of the error status code is * included. */ static void write_bin_error(conn *c, protocol_binary_response_status err, const char *errstr, int swallow) { size_t len; if (!errstr) { switch (err) { case PROTOCOL_BINARY_RESPONSE_ENOMEM: errstr = "Out of memory"; break; case PROTOCOL_BINARY_RESPONSE_UNKNOWN_COMMAND: errstr = "Unknown command"; break; case PROTOCOL_BINARY_RESPONSE_KEY_ENOENT: errstr = "Not found"; break; case PROTOCOL_BINARY_RESPONSE_EINVAL: errstr = "Invalid arguments"; break; case PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS: errstr = "Data exists for key."; break; case PROTOCOL_BINARY_RESPONSE_E2BIG: errstr = "Too large."; break; case PROTOCOL_BINARY_RESPONSE_DELTA_BADVAL: errstr = "Non-numeric server-side value for incr or decr"; break; case PROTOCOL_BINARY_RESPONSE_NOT_STORED: errstr = "Not stored."; break; case PROTOCOL_BINARY_RESPONSE_AUTH_ERROR: errstr = "Auth failure."; break; default: assert(false); errstr = "UNHANDLED ERROR"; fprintf(stderr, ">%d UNHANDLED ERROR: %d\n", c->sfd, err); } } if (settings.verbose > 1) { fprintf(stderr, ">%d Writing an error: %s\n", c->sfd, errstr); } len = strlen(errstr); add_bin_header(c, err, 0, 0, len); if (len > 0) { add_iov(c, errstr, len); } conn_set_state(c, conn_mwrite); if(swallow > 0) { c->sbytes = swallow; c->write_and_go = conn_swallow; } else { c->write_and_go = conn_new_cmd; } } /* Form and send a response to a command over the binary protocol */ static void write_bin_response(conn *c, void *d, int hlen, int keylen, int dlen) { if (!c->noreply || c->cmd == PROTOCOL_BINARY_CMD_GET || c->cmd == PROTOCOL_BINARY_CMD_GETK) { add_bin_header(c, 0, hlen, keylen, dlen); if(dlen > 0) { add_iov(c, d, dlen); } conn_set_state(c, conn_mwrite); c->write_and_go = conn_new_cmd; } else { conn_set_state(c, conn_new_cmd); } } static void complete_incr_bin(conn *c) { item *it; char *key; size_t nkey; /* Weird magic in add_delta forces me to pad here */ char tmpbuf[INCR_MAX_STORAGE_LEN]; uint64_t cas = 0; protocol_binary_response_incr* rsp = (protocol_binary_response_incr*)c->wbuf; protocol_binary_request_incr* req = binary_get_request(c); assert(c != NULL); assert(c->wsize >= sizeof(*rsp)); /* fix byteorder in the request */ req->message.body.delta = ntohll(req->message.body.delta); req->message.body.initial = ntohll(req->message.body.initial); req->message.body.expiration = ntohl(req->message.body.expiration); key = binary_get_key(c); nkey = c->binary_header.request.keylen; if (settings.verbose > 1) { int i; fprintf(stderr, "incr "); for (i = 0; i < nkey; i++) { fprintf(stderr, "%c", key[i]); } fprintf(stderr, " %lld, %llu, %d\n", (long long)req->message.body.delta, (long long)req->message.body.initial, req->message.body.expiration); } if (c->binary_header.request.cas != 0) { cas = c->binary_header.request.cas; } switch(add_delta(c, key, nkey, c->cmd == PROTOCOL_BINARY_CMD_INCREMENT, req->message.body.delta, tmpbuf, &cas)) { case OK: rsp->message.body.value = htonll(strtoull(tmpbuf, NULL, 10)); if (cas) { c->cas = cas; } write_bin_response(c, &rsp->message.body, 0, 0, sizeof(rsp->message.body.value)); break; case NON_NUMERIC: write_bin_error(c, PROTOCOL_BINARY_RESPONSE_DELTA_BADVAL, NULL, 0); break; case EOM: out_of_memory(c, "SERVER_ERROR Out of memory incrementing value"); break; case DELTA_ITEM_NOT_FOUND: if (req->message.body.expiration != 0xffffffff) { /* Save some room for the response */ rsp->message.body.value = htonll(req->message.body.initial); snprintf(tmpbuf, INCR_MAX_STORAGE_LEN, "%llu", (unsigned long long)req->message.body.initial); int res = strlen(tmpbuf); it = item_alloc(key, nkey, 0, realtime(req->message.body.expiration), res + 2); if (it != NULL) { memcpy(ITEM_data(it), tmpbuf, res); memcpy(ITEM_data(it) + res, "\r\n", 2); if (store_item(it, NREAD_ADD, c)) { c->cas = ITEM_get_cas(it); write_bin_response(c, &rsp->message.body, 0, 0, sizeof(rsp->message.body.value)); } else { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_NOT_STORED, NULL, 0); } item_remove(it); /* release our reference */ } else { out_of_memory(c, "SERVER_ERROR Out of memory allocating new item"); } } else { pthread_mutex_lock(&c->thread->stats.mutex); if (c->cmd == PROTOCOL_BINARY_CMD_INCREMENT) { c->thread->stats.incr_misses++; } else { c->thread->stats.decr_misses++; } pthread_mutex_unlock(&c->thread->stats.mutex); write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, NULL, 0); } break; case DELTA_ITEM_CAS_MISMATCH: write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, NULL, 0); break; } } static void complete_update_bin(conn *c) { protocol_binary_response_status eno = PROTOCOL_BINARY_RESPONSE_EINVAL; enum store_item_type ret = NOT_STORED; assert(c != NULL); item *it = c->item; pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.slab_stats[ITEM_clsid(it)].set_cmds++; pthread_mutex_unlock(&c->thread->stats.mutex); /* We don't actually receive the trailing two characters in the bin * protocol, so we're going to just set them here */ if ((it->it_flags & ITEM_CHUNKED) == 0) { *(ITEM_data(it) + it->nbytes - 2) = '\r'; *(ITEM_data(it) + it->nbytes - 1) = '\n'; } else { assert(c->ritem); item_chunk *ch = (item_chunk *) c->ritem; if (ch->size == ch->used) ch = ch->next; assert(ch->size - ch->used >= 2); ch->data[ch->used] = '\r'; ch->data[ch->used + 1] = '\n'; ch->used += 2; } ret = store_item(it, c->cmd, c); #ifdef ENABLE_DTRACE uint64_t cas = ITEM_get_cas(it); switch (c->cmd) { case NREAD_ADD: MEMCACHED_COMMAND_ADD(c->sfd, ITEM_key(it), it->nkey, (ret == STORED) ? it->nbytes : -1, cas); break; case NREAD_REPLACE: MEMCACHED_COMMAND_REPLACE(c->sfd, ITEM_key(it), it->nkey, (ret == STORED) ? it->nbytes : -1, cas); break; case NREAD_APPEND: MEMCACHED_COMMAND_APPEND(c->sfd, ITEM_key(it), it->nkey, (ret == STORED) ? it->nbytes : -1, cas); break; case NREAD_PREPEND: MEMCACHED_COMMAND_PREPEND(c->sfd, ITEM_key(it), it->nkey, (ret == STORED) ? it->nbytes : -1, cas); break; case NREAD_SET: MEMCACHED_COMMAND_SET(c->sfd, ITEM_key(it), it->nkey, (ret == STORED) ? it->nbytes : -1, cas); break; } #endif switch (ret) { case STORED: /* Stored */ write_bin_response(c, NULL, 0, 0, 0); break; case EXISTS: write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, NULL, 0); break; case NOT_FOUND: write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, NULL, 0); break; case NOT_STORED: case TOO_LARGE: case NO_MEMORY: if (c->cmd == NREAD_ADD) { eno = PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS; } else if(c->cmd == NREAD_REPLACE) { eno = PROTOCOL_BINARY_RESPONSE_KEY_ENOENT; } else { eno = PROTOCOL_BINARY_RESPONSE_NOT_STORED; } write_bin_error(c, eno, NULL, 0); } item_remove(c->item); /* release the c->item reference */ c->item = 0; } static void write_bin_miss_response(conn *c, char *key, size_t nkey) { if (nkey) { char *ofs = c->wbuf + sizeof(protocol_binary_response_header); add_bin_header(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, 0, nkey, nkey); memcpy(ofs, key, nkey); add_iov(c, ofs, nkey); conn_set_state(c, conn_mwrite); c->write_and_go = conn_new_cmd; } else { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, NULL, 0); } } static void process_bin_get_or_touch(conn *c) { item *it; protocol_binary_response_get* rsp = (protocol_binary_response_get*)c->wbuf; char* key = binary_get_key(c); size_t nkey = c->binary_header.request.keylen; int should_touch = (c->cmd == PROTOCOL_BINARY_CMD_TOUCH || c->cmd == PROTOCOL_BINARY_CMD_GAT || c->cmd == PROTOCOL_BINARY_CMD_GATK); int should_return_key = (c->cmd == PROTOCOL_BINARY_CMD_GETK || c->cmd == PROTOCOL_BINARY_CMD_GATK); int should_return_value = (c->cmd != PROTOCOL_BINARY_CMD_TOUCH); bool failed = false; if (settings.verbose > 1) { fprintf(stderr, "<%d %s ", c->sfd, should_touch ? "TOUCH" : "GET"); if (fwrite(key, 1, nkey, stderr)) {} fputc('\n', stderr); } if (should_touch) { protocol_binary_request_touch *t = binary_get_request(c); time_t exptime = ntohl(t->message.body.expiration); it = item_touch(key, nkey, realtime(exptime), c); } else { it = item_get(key, nkey, c, DO_UPDATE); } if (it) { /* the length has two unnecessary bytes ("\r\n") */ uint16_t keylen = 0; uint32_t bodylen = sizeof(rsp->message.body) + (it->nbytes - 2); pthread_mutex_lock(&c->thread->stats.mutex); if (should_touch) { c->thread->stats.touch_cmds++; c->thread->stats.slab_stats[ITEM_clsid(it)].touch_hits++; } else { c->thread->stats.get_cmds++; c->thread->stats.lru_hits[it->slabs_clsid]++; } pthread_mutex_unlock(&c->thread->stats.mutex); if (should_touch) { MEMCACHED_COMMAND_TOUCH(c->sfd, ITEM_key(it), it->nkey, it->nbytes, ITEM_get_cas(it)); } else { MEMCACHED_COMMAND_GET(c->sfd, ITEM_key(it), it->nkey, it->nbytes, ITEM_get_cas(it)); } if (c->cmd == PROTOCOL_BINARY_CMD_TOUCH) { bodylen -= it->nbytes - 2; } else if (should_return_key) { bodylen += nkey; keylen = nkey; } add_bin_header(c, 0, sizeof(rsp->message.body), keylen, bodylen); rsp->message.header.response.cas = htonll(ITEM_get_cas(it)); // add the flags FLAGS_CONV(it, rsp->message.body.flags); rsp->message.body.flags = htonl(rsp->message.body.flags); add_iov(c, &rsp->message.body, sizeof(rsp->message.body)); if (should_return_key) { add_iov(c, ITEM_key(it), nkey); } if (should_return_value) { /* Add the data minus the CRLF */ #ifdef EXTSTORE if (it->it_flags & ITEM_HDR) { int iovcnt = 4; int iovst = c->iovused - 3; if (!should_return_key) { iovcnt = 3; iovst = c->iovused - 2; } if (_get_extstore(c, it, iovst, iovcnt) != 0) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.get_oom_extstore++; pthread_mutex_unlock(&c->thread->stats.mutex); failed = true; } } else if ((it->it_flags & ITEM_CHUNKED) == 0) { add_iov(c, ITEM_data(it), it->nbytes - 2); } else { add_chunked_item_iovs(c, it, it->nbytes - 2); } #else if ((it->it_flags & ITEM_CHUNKED) == 0) { add_iov(c, ITEM_data(it), it->nbytes - 2); } else { add_chunked_item_iovs(c, it, it->nbytes - 2); } #endif } if (!failed) { conn_set_state(c, conn_mwrite); c->write_and_go = conn_new_cmd; /* Remember this command so we can garbage collect it later */ #ifdef EXTSTORE if ((it->it_flags & ITEM_HDR) != 0 && should_return_value) { // Only have extstore clean if header and returning value. c->item = NULL; } else { c->item = it; } #else c->item = it; #endif } else { item_remove(it); } } else { failed = true; } if (failed) { pthread_mutex_lock(&c->thread->stats.mutex); if (should_touch) { c->thread->stats.touch_cmds++; c->thread->stats.touch_misses++; } else { c->thread->stats.get_cmds++; c->thread->stats.get_misses++; } pthread_mutex_unlock(&c->thread->stats.mutex); if (should_touch) { MEMCACHED_COMMAND_TOUCH(c->sfd, key, nkey, -1, 0); } else { MEMCACHED_COMMAND_GET(c->sfd, key, nkey, -1, 0); } if (c->noreply) { conn_set_state(c, conn_new_cmd); } else { if (should_return_key) { write_bin_miss_response(c, key, nkey); } else { write_bin_miss_response(c, NULL, 0); } } } if (settings.detail_enabled) { stats_prefix_record_get(key, nkey, NULL != it); } } static void append_bin_stats(const char *key, const uint16_t klen, const char *val, const uint32_t vlen, conn *c) { char *buf = c->stats.buffer + c->stats.offset; uint32_t bodylen = klen + vlen; protocol_binary_response_header header = { .response.magic = (uint8_t)PROTOCOL_BINARY_RES, .response.opcode = PROTOCOL_BINARY_CMD_STAT, .response.keylen = (uint16_t)htons(klen), .response.datatype = (uint8_t)PROTOCOL_BINARY_RAW_BYTES, .response.bodylen = htonl(bodylen), .response.opaque = c->opaque }; memcpy(buf, header.bytes, sizeof(header.response)); buf += sizeof(header.response); if (klen > 0) { memcpy(buf, key, klen); buf += klen; if (vlen > 0) { memcpy(buf, val, vlen); } } c->stats.offset += sizeof(header.response) + bodylen; } static void append_ascii_stats(const char *key, const uint16_t klen, const char *val, const uint32_t vlen, conn *c) { char *pos = c->stats.buffer + c->stats.offset; uint32_t nbytes = 0; int remaining = c->stats.size - c->stats.offset; int room = remaining - 1; if (klen == 0 && vlen == 0) { nbytes = snprintf(pos, room, "END\r\n"); } else if (vlen == 0) { nbytes = snprintf(pos, room, "STAT %s\r\n", key); } else { nbytes = snprintf(pos, room, "STAT %s %s\r\n", key, val); } c->stats.offset += nbytes; } static bool grow_stats_buf(conn *c, size_t needed) { size_t nsize = c->stats.size; size_t available = nsize - c->stats.offset; bool rv = true; /* Special case: No buffer -- need to allocate fresh */ if (c->stats.buffer == NULL) { nsize = 1024; available = c->stats.size = c->stats.offset = 0; } while (needed > available) { assert(nsize > 0); nsize = nsize << 1; available = nsize - c->stats.offset; } if (nsize != c->stats.size) { char *ptr = realloc(c->stats.buffer, nsize); if (ptr) { c->stats.buffer = ptr; c->stats.size = nsize; } else { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); rv = false; } } return rv; } static void append_stats(const char *key, const uint16_t klen, const char *val, const uint32_t vlen, const void *cookie) { /* value without a key is invalid */ if (klen == 0 && vlen > 0) { return; } conn *c = (conn*)cookie; if (c->protocol == binary_prot) { size_t needed = vlen + klen + sizeof(protocol_binary_response_header); if (!grow_stats_buf(c, needed)) { return; } append_bin_stats(key, klen, val, vlen, c); } else { size_t needed = vlen + klen + 10; // 10 == "STAT = \r\n" if (!grow_stats_buf(c, needed)) { return; } append_ascii_stats(key, klen, val, vlen, c); } assert(c->stats.offset <= c->stats.size); } static void process_bin_stat(conn *c) { char *subcommand = binary_get_key(c); size_t nkey = c->binary_header.request.keylen; if (settings.verbose > 1) { int ii; fprintf(stderr, "<%d STATS ", c->sfd); for (ii = 0; ii < nkey; ++ii) { fprintf(stderr, "%c", subcommand[ii]); } fprintf(stderr, "\n"); } if (nkey == 0) { /* request all statistics */ server_stats(&append_stats, c); (void)get_stats(NULL, 0, &append_stats, c); } else if (strncmp(subcommand, "reset", 5) == 0) { stats_reset(); } else if (strncmp(subcommand, "settings", 8) == 0) { process_stat_settings(&append_stats, c); } else if (strncmp(subcommand, "detail", 6) == 0) { char *subcmd_pos = subcommand + 6; if (strncmp(subcmd_pos, " dump", 5) == 0) { int len; char *dump_buf = stats_prefix_dump(&len); if (dump_buf == NULL || len <= 0) { out_of_memory(c, "SERVER_ERROR Out of memory generating stats"); if (dump_buf != NULL) free(dump_buf); return; } else { append_stats("detailed", strlen("detailed"), dump_buf, len, c); free(dump_buf); } } else if (strncmp(subcmd_pos, " on", 3) == 0) { settings.detail_enabled = 1; } else if (strncmp(subcmd_pos, " off", 4) == 0) { settings.detail_enabled = 0; } else { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, NULL, 0); return; } } else { if (get_stats(subcommand, nkey, &append_stats, c)) { if (c->stats.buffer == NULL) { out_of_memory(c, "SERVER_ERROR Out of memory generating stats"); } else { write_and_free(c, c->stats.buffer, c->stats.offset); c->stats.buffer = NULL; } } else { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, NULL, 0); } return; } /* Append termination package and start the transfer */ append_stats(NULL, 0, NULL, 0, c); if (c->stats.buffer == NULL) { out_of_memory(c, "SERVER_ERROR Out of memory preparing to send stats"); } else { write_and_free(c, c->stats.buffer, c->stats.offset); c->stats.buffer = NULL; } } static void bin_read_key(conn *c, enum bin_substates next_substate, int extra) { assert(c); c->substate = next_substate; c->rlbytes = c->keylen + extra; /* Ok... do we have room for the extras and the key in the input buffer? */ ptrdiff_t offset = c->rcurr + sizeof(protocol_binary_request_header) - c->rbuf; if (c->rlbytes > c->rsize - offset) { size_t nsize = c->rsize; size_t size = c->rlbytes + sizeof(protocol_binary_request_header); while (size > nsize) { nsize *= 2; } if (nsize != c->rsize) { if (settings.verbose > 1) { fprintf(stderr, "%d: Need to grow buffer from %lu to %lu\n", c->sfd, (unsigned long)c->rsize, (unsigned long)nsize); } char *newm = realloc(c->rbuf, nsize); if (newm == NULL) { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); if (settings.verbose) { fprintf(stderr, "%d: Failed to grow buffer.. closing connection\n", c->sfd); } conn_set_state(c, conn_closing); return; } c->rbuf= newm; /* rcurr should point to the same offset in the packet */ c->rcurr = c->rbuf + offset - sizeof(protocol_binary_request_header); c->rsize = nsize; } if (c->rbuf != c->rcurr) { memmove(c->rbuf, c->rcurr, c->rbytes); c->rcurr = c->rbuf; if (settings.verbose > 1) { fprintf(stderr, "%d: Repack input buffer\n", c->sfd); } } } /* preserve the header in the buffer.. */ c->ritem = c->rcurr + sizeof(protocol_binary_request_header); conn_set_state(c, conn_nread); } /* Just write an error message and disconnect the client */ static void handle_binary_protocol_error(conn *c) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_EINVAL, NULL, 0); if (settings.verbose) { fprintf(stderr, "Protocol error (opcode %02x), close connection %d\n", c->binary_header.request.opcode, c->sfd); } c->write_and_go = conn_closing; } static void init_sasl_conn(conn *c) { assert(c); /* should something else be returned? */ if (!settings.sasl) return; c->authenticated = false; if (!c->sasl_conn) { int result=sasl_server_new("memcached", NULL, my_sasl_hostname[0] ? my_sasl_hostname : NULL, NULL, NULL, NULL, 0, &c->sasl_conn); if (result != SASL_OK) { if (settings.verbose) { fprintf(stderr, "Failed to initialize SASL conn.\n"); } c->sasl_conn = NULL; } } } static void bin_list_sasl_mechs(conn *c) { // Guard against a disabled SASL. if (!settings.sasl) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_UNKNOWN_COMMAND, NULL, c->binary_header.request.bodylen - c->binary_header.request.keylen); return; } init_sasl_conn(c); const char *result_string = NULL; unsigned int string_length = 0; int result=sasl_listmech(c->sasl_conn, NULL, "", /* What to prepend the string with */ " ", /* What to separate mechanisms with */ "", /* What to append to the string */ &result_string, &string_length, NULL); if (result != SASL_OK) { /* Perhaps there's a better error for this... */ if (settings.verbose) { fprintf(stderr, "Failed to list SASL mechanisms.\n"); } write_bin_error(c, PROTOCOL_BINARY_RESPONSE_AUTH_ERROR, NULL, 0); return; } write_bin_response(c, (char*)result_string, 0, 0, string_length); } static void process_bin_sasl_auth(conn *c) { // Guard for handling disabled SASL on the server. if (!settings.sasl) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_UNKNOWN_COMMAND, NULL, c->binary_header.request.bodylen - c->binary_header.request.keylen); return; } assert(c->binary_header.request.extlen == 0); int nkey = c->binary_header.request.keylen; int vlen = c->binary_header.request.bodylen - nkey; if (nkey > MAX_SASL_MECH_LEN) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_EINVAL, NULL, vlen); c->write_and_go = conn_swallow; return; } char *key = binary_get_key(c); assert(key); item *it = item_alloc(key, nkey, 0, 0, vlen+2); /* Can't use a chunked item for SASL authentication. */ if (it == 0 || (it->it_flags & ITEM_CHUNKED)) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_ENOMEM, NULL, vlen); c->write_and_go = conn_swallow; return; } c->item = it; c->ritem = ITEM_data(it); c->rlbytes = vlen; conn_set_state(c, conn_nread); c->substate = bin_reading_sasl_auth_data; } static void process_bin_complete_sasl_auth(conn *c) { assert(settings.sasl); const char *out = NULL; unsigned int outlen = 0; assert(c->item); init_sasl_conn(c); int nkey = c->binary_header.request.keylen; int vlen = c->binary_header.request.bodylen - nkey; if (nkey > ((item*) c->item)->nkey) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_EINVAL, NULL, vlen); c->write_and_go = conn_swallow; item_unlink(c->item); return; } char mech[nkey+1]; memcpy(mech, ITEM_key((item*)c->item), nkey); mech[nkey] = 0x00; if (settings.verbose) fprintf(stderr, "mech: ``%s'' with %d bytes of data\n", mech, vlen); const char *challenge = vlen == 0 ? NULL : ITEM_data((item*) c->item); if (vlen > ((item*) c->item)->nbytes) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_EINVAL, NULL, vlen); c->write_and_go = conn_swallow; item_unlink(c->item); return; } int result=-1; switch (c->cmd) { case PROTOCOL_BINARY_CMD_SASL_AUTH: result = sasl_server_start(c->sasl_conn, mech, challenge, vlen, &out, &outlen); c->sasl_started = (result == SASL_OK || result == SASL_CONTINUE); break; case PROTOCOL_BINARY_CMD_SASL_STEP: if (!c->sasl_started) { if (settings.verbose) { fprintf(stderr, "%d: SASL_STEP called but sasl_server_start " "not called for this connection!\n", c->sfd); } break; } result = sasl_server_step(c->sasl_conn, challenge, vlen, &out, &outlen); break; default: assert(false); /* CMD should be one of the above */ /* This code is pretty much impossible, but makes the compiler happier */ if (settings.verbose) { fprintf(stderr, "Unhandled command %d with challenge %s\n", c->cmd, challenge); } break; } item_unlink(c->item); if (settings.verbose) { fprintf(stderr, "sasl result code: %d\n", result); } switch(result) { case SASL_OK: c->authenticated = true; write_bin_response(c, "Authenticated", 0, 0, strlen("Authenticated")); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.auth_cmds++; pthread_mutex_unlock(&c->thread->stats.mutex); break; case SASL_CONTINUE: add_bin_header(c, PROTOCOL_BINARY_RESPONSE_AUTH_CONTINUE, 0, 0, outlen); if(outlen > 0) { add_iov(c, out, outlen); } conn_set_state(c, conn_mwrite); c->write_and_go = conn_new_cmd; break; default: if (settings.verbose) fprintf(stderr, "Unknown sasl response: %d\n", result); write_bin_error(c, PROTOCOL_BINARY_RESPONSE_AUTH_ERROR, NULL, 0); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.auth_cmds++; c->thread->stats.auth_errors++; pthread_mutex_unlock(&c->thread->stats.mutex); } } static bool authenticated(conn *c) { assert(settings.sasl); bool rv = false; switch (c->cmd) { case PROTOCOL_BINARY_CMD_SASL_LIST_MECHS: /* FALLTHROUGH */ case PROTOCOL_BINARY_CMD_SASL_AUTH: /* FALLTHROUGH */ case PROTOCOL_BINARY_CMD_SASL_STEP: /* FALLTHROUGH */ case PROTOCOL_BINARY_CMD_VERSION: /* FALLTHROUGH */ rv = true; break; default: rv = c->authenticated; } if (settings.verbose > 1) { fprintf(stderr, "authenticated() in cmd 0x%02x is %s\n", c->cmd, rv ? "true" : "false"); } return rv; } static void dispatch_bin_command(conn *c) { int protocol_error = 0; uint8_t extlen = c->binary_header.request.extlen; uint16_t keylen = c->binary_header.request.keylen; uint32_t bodylen = c->binary_header.request.bodylen; if (keylen > bodylen || keylen + extlen > bodylen) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_UNKNOWN_COMMAND, NULL, 0); c->write_and_go = conn_closing; return; } if (settings.sasl && !authenticated(c)) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_AUTH_ERROR, NULL, 0); c->write_and_go = conn_closing; return; } MEMCACHED_PROCESS_COMMAND_START(c->sfd, c->rcurr, c->rbytes); c->noreply = true; /* binprot supports 16bit keys, but internals are still 8bit */ if (keylen > KEY_MAX_LENGTH) { handle_binary_protocol_error(c); return; } switch (c->cmd) { case PROTOCOL_BINARY_CMD_SETQ: c->cmd = PROTOCOL_BINARY_CMD_SET; break; case PROTOCOL_BINARY_CMD_ADDQ: c->cmd = PROTOCOL_BINARY_CMD_ADD; break; case PROTOCOL_BINARY_CMD_REPLACEQ: c->cmd = PROTOCOL_BINARY_CMD_REPLACE; break; case PROTOCOL_BINARY_CMD_DELETEQ: c->cmd = PROTOCOL_BINARY_CMD_DELETE; break; case PROTOCOL_BINARY_CMD_INCREMENTQ: c->cmd = PROTOCOL_BINARY_CMD_INCREMENT; break; case PROTOCOL_BINARY_CMD_DECREMENTQ: c->cmd = PROTOCOL_BINARY_CMD_DECREMENT; break; case PROTOCOL_BINARY_CMD_QUITQ: c->cmd = PROTOCOL_BINARY_CMD_QUIT; break; case PROTOCOL_BINARY_CMD_FLUSHQ: c->cmd = PROTOCOL_BINARY_CMD_FLUSH; break; case PROTOCOL_BINARY_CMD_APPENDQ: c->cmd = PROTOCOL_BINARY_CMD_APPEND; break; case PROTOCOL_BINARY_CMD_PREPENDQ: c->cmd = PROTOCOL_BINARY_CMD_PREPEND; break; case PROTOCOL_BINARY_CMD_GETQ: c->cmd = PROTOCOL_BINARY_CMD_GET; break; case PROTOCOL_BINARY_CMD_GETKQ: c->cmd = PROTOCOL_BINARY_CMD_GETK; break; case PROTOCOL_BINARY_CMD_GATQ: c->cmd = PROTOCOL_BINARY_CMD_GAT; break; case PROTOCOL_BINARY_CMD_GATKQ: c->cmd = PROTOCOL_BINARY_CMD_GATK; break; default: c->noreply = false; } switch (c->cmd) { case PROTOCOL_BINARY_CMD_VERSION: if (extlen == 0 && keylen == 0 && bodylen == 0) { write_bin_response(c, VERSION, 0, 0, strlen(VERSION)); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_FLUSH: if (keylen == 0 && bodylen == extlen && (extlen == 0 || extlen == 4)) { bin_read_key(c, bin_read_flush_exptime, extlen); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_NOOP: if (extlen == 0 && keylen == 0 && bodylen == 0) { write_bin_response(c, NULL, 0, 0, 0); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_SET: /* FALLTHROUGH */ case PROTOCOL_BINARY_CMD_ADD: /* FALLTHROUGH */ case PROTOCOL_BINARY_CMD_REPLACE: if (extlen == 8 && keylen != 0 && bodylen >= (keylen + 8)) { bin_read_key(c, bin_reading_set_header, 8); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_GETQ: /* FALLTHROUGH */ case PROTOCOL_BINARY_CMD_GET: /* FALLTHROUGH */ case PROTOCOL_BINARY_CMD_GETKQ: /* FALLTHROUGH */ case PROTOCOL_BINARY_CMD_GETK: if (extlen == 0 && bodylen == keylen && keylen > 0) { bin_read_key(c, bin_reading_get_key, 0); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_DELETE: if (keylen > 0 && extlen == 0 && bodylen == keylen) { bin_read_key(c, bin_reading_del_header, extlen); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_INCREMENT: case PROTOCOL_BINARY_CMD_DECREMENT: if (keylen > 0 && extlen == 20 && bodylen == (keylen + extlen)) { bin_read_key(c, bin_reading_incr_header, 20); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_APPEND: case PROTOCOL_BINARY_CMD_PREPEND: if (keylen > 0 && extlen == 0) { bin_read_key(c, bin_reading_set_header, 0); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_STAT: if (extlen == 0) { bin_read_key(c, bin_reading_stat, 0); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_QUIT: if (keylen == 0 && extlen == 0 && bodylen == 0) { write_bin_response(c, NULL, 0, 0, 0); c->write_and_go = conn_closing; if (c->noreply) { conn_set_state(c, conn_closing); } } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_SASL_LIST_MECHS: if (extlen == 0 && keylen == 0 && bodylen == 0) { bin_list_sasl_mechs(c); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_SASL_AUTH: case PROTOCOL_BINARY_CMD_SASL_STEP: if (extlen == 0 && keylen != 0) { bin_read_key(c, bin_reading_sasl_auth, 0); } else { protocol_error = 1; } break; case PROTOCOL_BINARY_CMD_TOUCH: case PROTOCOL_BINARY_CMD_GAT: case PROTOCOL_BINARY_CMD_GATQ: case PROTOCOL_BINARY_CMD_GATK: case PROTOCOL_BINARY_CMD_GATKQ: if (extlen == 4 && keylen != 0) { bin_read_key(c, bin_reading_touch_key, 4); } else { protocol_error = 1; } break; default: write_bin_error(c, PROTOCOL_BINARY_RESPONSE_UNKNOWN_COMMAND, NULL, bodylen); } if (protocol_error) handle_binary_protocol_error(c); } static void process_bin_update(conn *c) { char *key; int nkey; int vlen; item *it; protocol_binary_request_set* req = binary_get_request(c); assert(c != NULL); key = binary_get_key(c); nkey = c->binary_header.request.keylen; /* fix byteorder in the request */ req->message.body.flags = ntohl(req->message.body.flags); req->message.body.expiration = ntohl(req->message.body.expiration); vlen = c->binary_header.request.bodylen - (nkey + c->binary_header.request.extlen); if (settings.verbose > 1) { int ii; if (c->cmd == PROTOCOL_BINARY_CMD_ADD) { fprintf(stderr, "<%d ADD ", c->sfd); } else if (c->cmd == PROTOCOL_BINARY_CMD_SET) { fprintf(stderr, "<%d SET ", c->sfd); } else { fprintf(stderr, "<%d REPLACE ", c->sfd); } for (ii = 0; ii < nkey; ++ii) { fprintf(stderr, "%c", key[ii]); } fprintf(stderr, " Value len is %d", vlen); fprintf(stderr, "\n"); } if (settings.detail_enabled) { stats_prefix_record_set(key, nkey); } it = item_alloc(key, nkey, req->message.body.flags, realtime(req->message.body.expiration), vlen+2); if (it == 0) { enum store_item_type status; if (! item_size_ok(nkey, req->message.body.flags, vlen + 2)) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_E2BIG, NULL, vlen); status = TOO_LARGE; } else { out_of_memory(c, "SERVER_ERROR Out of memory allocating item"); /* This error generating method eats the swallow value. Add here. */ c->sbytes = vlen; status = NO_MEMORY; } /* FIXME: losing c->cmd since it's translated below. refactor? */ LOGGER_LOG(c->thread->l, LOG_MUTATIONS, LOGGER_ITEM_STORE, NULL, status, 0, key, nkey, req->message.body.expiration, ITEM_clsid(it), c->sfd); /* Avoid stale data persisting in cache because we failed alloc. * Unacceptable for SET. Anywhere else too? */ if (c->cmd == PROTOCOL_BINARY_CMD_SET) { it = item_get(key, nkey, c, DONT_UPDATE); if (it) { item_unlink(it); STORAGE_delete(c->thread->storage, it); item_remove(it); } } /* swallow the data line */ c->write_and_go = conn_swallow; return; } ITEM_set_cas(it, c->binary_header.request.cas); switch (c->cmd) { case PROTOCOL_BINARY_CMD_ADD: c->cmd = NREAD_ADD; break; case PROTOCOL_BINARY_CMD_SET: c->cmd = NREAD_SET; break; case PROTOCOL_BINARY_CMD_REPLACE: c->cmd = NREAD_REPLACE; break; default: assert(0); } if (ITEM_get_cas(it) != 0) { c->cmd = NREAD_CAS; } c->item = it; #ifdef NEED_ALIGN if (it->it_flags & ITEM_CHUNKED) { c->ritem = ITEM_schunk(it); } else { c->ritem = ITEM_data(it); } #else c->ritem = ITEM_data(it); #endif c->rlbytes = vlen; conn_set_state(c, conn_nread); c->substate = bin_read_set_value; } static void process_bin_append_prepend(conn *c) { char *key; int nkey; int vlen; item *it; assert(c != NULL); key = binary_get_key(c); nkey = c->binary_header.request.keylen; vlen = c->binary_header.request.bodylen - nkey; if (settings.verbose > 1) { fprintf(stderr, "Value len is %d\n", vlen); } if (settings.detail_enabled) { stats_prefix_record_set(key, nkey); } it = item_alloc(key, nkey, 0, 0, vlen+2); if (it == 0) { if (! item_size_ok(nkey, 0, vlen + 2)) { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_E2BIG, NULL, vlen); } else { out_of_memory(c, "SERVER_ERROR Out of memory allocating item"); /* OOM calls eat the swallow value. Add here. */ c->sbytes = vlen; } /* swallow the data line */ c->write_and_go = conn_swallow; return; } ITEM_set_cas(it, c->binary_header.request.cas); switch (c->cmd) { case PROTOCOL_BINARY_CMD_APPEND: c->cmd = NREAD_APPEND; break; case PROTOCOL_BINARY_CMD_PREPEND: c->cmd = NREAD_PREPEND; break; default: assert(0); } c->item = it; #ifdef NEED_ALIGN if (it->it_flags & ITEM_CHUNKED) { c->ritem = ITEM_schunk(it); } else { c->ritem = ITEM_data(it); } #else c->ritem = ITEM_data(it); #endif c->rlbytes = vlen; conn_set_state(c, conn_nread); c->substate = bin_read_set_value; } static void process_bin_flush(conn *c) { time_t exptime = 0; protocol_binary_request_flush* req = binary_get_request(c); rel_time_t new_oldest = 0; if (!settings.flush_enabled) { // flush_all is not allowed but we log it on stats write_bin_error(c, PROTOCOL_BINARY_RESPONSE_AUTH_ERROR, NULL, 0); return; } if (c->binary_header.request.extlen == sizeof(req->message.body)) { exptime = ntohl(req->message.body.expiration); } if (exptime > 0) { new_oldest = realtime(exptime); } else { new_oldest = current_time; } if (settings.use_cas) { settings.oldest_live = new_oldest - 1; if (settings.oldest_live <= current_time) settings.oldest_cas = get_cas_id(); } else { settings.oldest_live = new_oldest; } pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.flush_cmds++; pthread_mutex_unlock(&c->thread->stats.mutex); write_bin_response(c, NULL, 0, 0, 0); } static void process_bin_delete(conn *c) { item *it; uint32_t hv; protocol_binary_request_delete* req = binary_get_request(c); char* key = binary_get_key(c); size_t nkey = c->binary_header.request.keylen; assert(c != NULL); if (settings.verbose > 1) { int ii; fprintf(stderr, "Deleting "); for (ii = 0; ii < nkey; ++ii) { fprintf(stderr, "%c", key[ii]); } fprintf(stderr, "\n"); } if (settings.detail_enabled) { stats_prefix_record_delete(key, nkey); } it = item_get_locked(key, nkey, c, DONT_UPDATE, &hv); if (it) { uint64_t cas = ntohll(req->message.header.request.cas); if (cas == 0 || cas == ITEM_get_cas(it)) { MEMCACHED_COMMAND_DELETE(c->sfd, ITEM_key(it), it->nkey); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.slab_stats[ITEM_clsid(it)].delete_hits++; pthread_mutex_unlock(&c->thread->stats.mutex); do_item_unlink(it, hv); STORAGE_delete(c->thread->storage, it); write_bin_response(c, NULL, 0, 0, 0); } else { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, NULL, 0); } do_item_remove(it); /* release our reference */ } else { write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, NULL, 0); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.delete_misses++; pthread_mutex_unlock(&c->thread->stats.mutex); } item_unlock(hv); } static void complete_nread_binary(conn *c) { assert(c != NULL); assert(c->cmd >= 0); switch(c->substate) { case bin_reading_set_header: if (c->cmd == PROTOCOL_BINARY_CMD_APPEND || c->cmd == PROTOCOL_BINARY_CMD_PREPEND) { process_bin_append_prepend(c); } else { process_bin_update(c); } break; case bin_read_set_value: complete_update_bin(c); break; case bin_reading_get_key: case bin_reading_touch_key: process_bin_get_or_touch(c); break; case bin_reading_stat: process_bin_stat(c); break; case bin_reading_del_header: process_bin_delete(c); break; case bin_reading_incr_header: complete_incr_bin(c); break; case bin_read_flush_exptime: process_bin_flush(c); break; case bin_reading_sasl_auth: process_bin_sasl_auth(c); break; case bin_reading_sasl_auth_data: process_bin_complete_sasl_auth(c); break; default: fprintf(stderr, "Not handling substate %d\n", c->substate); assert(0); } } static void reset_cmd_handler(conn *c) { c->cmd = -1; c->substate = bin_no_state; if(c->item != NULL) { item_remove(c->item); c->item = NULL; } conn_shrink(c); if (c->rbytes > 0) { conn_set_state(c, conn_parse_cmd); } else { conn_set_state(c, conn_waiting); } } static void complete_nread(conn *c) { assert(c != NULL); assert(c->protocol == ascii_prot || c->protocol == binary_prot); if (c->protocol == ascii_prot) { complete_nread_ascii(c); } else if (c->protocol == binary_prot) { complete_nread_binary(c); } } /* Destination must always be chunked */ /* This should be part of item.c */ static int _store_item_copy_chunks(item *d_it, item *s_it, const int len) { item_chunk *dch = (item_chunk *) ITEM_schunk(d_it); /* Advance dch until we find free space */ while (dch->size == dch->used) { if (dch->next) { dch = dch->next; } else { break; } } if (s_it->it_flags & ITEM_CHUNKED) { int remain = len; item_chunk *sch = (item_chunk *) ITEM_schunk(s_it); int copied = 0; /* Fills dch's to capacity, not straight copy sch in case data is * being added or removed (ie append/prepend) */ while (sch && dch && remain) { assert(dch->used <= dch->size); int todo = (dch->size - dch->used < sch->used - copied) ? dch->size - dch->used : sch->used - copied; if (remain < todo) todo = remain; memcpy(dch->data + dch->used, sch->data + copied, todo); dch->used += todo; copied += todo; remain -= todo; assert(dch->used <= dch->size); if (dch->size == dch->used) { item_chunk *tch = do_item_alloc_chunk(dch, remain); if (tch) { dch = tch; } else { return -1; } } assert(copied <= sch->used); if (copied == sch->used) { copied = 0; sch = sch->next; } } /* assert that the destination had enough space for the source */ assert(remain == 0); } else { int done = 0; /* Fill dch's via a non-chunked item. */ while (len > done && dch) { int todo = (dch->size - dch->used < len - done) ? dch->size - dch->used : len - done; //assert(dch->size - dch->used != 0); memcpy(dch->data + dch->used, ITEM_data(s_it) + done, todo); done += todo; dch->used += todo; assert(dch->used <= dch->size); if (dch->size == dch->used) { item_chunk *tch = do_item_alloc_chunk(dch, len - done); if (tch) { dch = tch; } else { return -1; } } } assert(len == done); } return 0; } static int _store_item_copy_data(int comm, item *old_it, item *new_it, item *add_it) { if (comm == NREAD_APPEND) { if (new_it->it_flags & ITEM_CHUNKED) { if (_store_item_copy_chunks(new_it, old_it, old_it->nbytes - 2) == -1 || _store_item_copy_chunks(new_it, add_it, add_it->nbytes) == -1) { return -1; } } else { memcpy(ITEM_data(new_it), ITEM_data(old_it), old_it->nbytes); memcpy(ITEM_data(new_it) + old_it->nbytes - 2 /* CRLF */, ITEM_data(add_it), add_it->nbytes); } } else { /* NREAD_PREPEND */ if (new_it->it_flags & ITEM_CHUNKED) { if (_store_item_copy_chunks(new_it, add_it, add_it->nbytes - 2) == -1 || _store_item_copy_chunks(new_it, old_it, old_it->nbytes) == -1) { return -1; } } else { memcpy(ITEM_data(new_it), ITEM_data(add_it), add_it->nbytes); memcpy(ITEM_data(new_it) + add_it->nbytes - 2 /* CRLF */, ITEM_data(old_it), old_it->nbytes); } } return 0; } /* * Stores an item in the cache according to the semantics of one of the set * commands. Protected by the item lock. * * Returns the state of storage. */ enum store_item_type do_store_item(item *it, int comm, conn *c, const uint32_t hv) { char *key = ITEM_key(it); item *old_it = do_item_get(key, it->nkey, hv, c, DONT_UPDATE); enum store_item_type stored = NOT_STORED; item *new_it = NULL; uint32_t flags; if (old_it != NULL && comm == NREAD_ADD) { /* add only adds a nonexistent item, but promote to head of LRU */ do_item_update(old_it); } else if (!old_it && (comm == NREAD_REPLACE || comm == NREAD_APPEND || comm == NREAD_PREPEND)) { /* replace only replaces an existing value; don't store */ } else if (comm == NREAD_CAS) { /* validate cas operation */ if (old_it == NULL) { // LRU expired stored = NOT_FOUND; pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.cas_misses++; pthread_mutex_unlock(&c->thread->stats.mutex); } else if (ITEM_get_cas(it) == ITEM_get_cas(old_it)) { // cas validates // it and old_it may belong to different classes. // I'm updating the stats for the one that's getting pushed out pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.slab_stats[ITEM_clsid(old_it)].cas_hits++; pthread_mutex_unlock(&c->thread->stats.mutex); STORAGE_delete(c->thread->storage, old_it); item_replace(old_it, it, hv); stored = STORED; } else if (c->set_stale && ITEM_get_cas(it) < ITEM_get_cas(old_it)) { // if we're allowed to set a stale value, CAS must be lower than // the current item's CAS. // This replaces the value, but should preserve TTL, and stale // item marker bit + token sent if exists. it->exptime = old_it->exptime; it->it_flags |= ITEM_STALE; if (old_it->it_flags & ITEM_TOKEN_SENT) { it->it_flags |= ITEM_TOKEN_SENT; } pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.slab_stats[ITEM_clsid(old_it)].cas_hits++; pthread_mutex_unlock(&c->thread->stats.mutex); STORAGE_delete(c->thread->storage, old_it); item_replace(old_it, it, hv); stored = STORED; } else { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.slab_stats[ITEM_clsid(old_it)].cas_badval++; pthread_mutex_unlock(&c->thread->stats.mutex); if(settings.verbose > 1) { fprintf(stderr, "CAS: failure: expected %llu, got %llu\n", (unsigned long long)ITEM_get_cas(old_it), (unsigned long long)ITEM_get_cas(it)); } stored = EXISTS; } } else { int failed_alloc = 0; /* * Append - combine new and old record into single one. Here it's * atomic and thread-safe. */ if (comm == NREAD_APPEND || comm == NREAD_PREPEND) { /* * Validate CAS */ if (ITEM_get_cas(it) != 0) { // CAS much be equal if (ITEM_get_cas(it) != ITEM_get_cas(old_it)) { stored = EXISTS; } } #ifdef EXTSTORE if ((old_it->it_flags & ITEM_HDR) != 0) { /* block append/prepend from working with extstore-d items. * also don't replace the header with the append chunk * accidentally, so mark as a failed_alloc. */ failed_alloc = 1; } else #endif if (stored == NOT_STORED) { /* we have it and old_it here - alloc memory to hold both */ FLAGS_CONV(old_it, flags); new_it = do_item_alloc(key, it->nkey, flags, old_it->exptime, it->nbytes + old_it->nbytes - 2 /* CRLF */); /* copy data from it and old_it to new_it */ if (new_it == NULL || _store_item_copy_data(comm, old_it, new_it, it) == -1) { failed_alloc = 1; stored = NOT_STORED; // failed data copy, free up. if (new_it != NULL) item_remove(new_it); } else { it = new_it; } } } if (stored == NOT_STORED && failed_alloc == 0) { if (old_it != NULL) { STORAGE_delete(c->thread->storage, old_it); item_replace(old_it, it, hv); } else { do_item_link(it, hv); } c->cas = ITEM_get_cas(it); stored = STORED; } } if (old_it != NULL) do_item_remove(old_it); /* release our reference */ if (new_it != NULL) do_item_remove(new_it); if (stored == STORED) { c->cas = ITEM_get_cas(it); } LOGGER_LOG(c->thread->l, LOG_MUTATIONS, LOGGER_ITEM_STORE, NULL, stored, comm, ITEM_key(it), it->nkey, it->exptime, ITEM_clsid(it), c->sfd); return stored; } typedef struct token_s { char *value; size_t length; } token_t; #define COMMAND_TOKEN 0 #define SUBCOMMAND_TOKEN 1 #define KEY_TOKEN 1 #define MAX_TOKENS 8 /* * Tokenize the command string by replacing whitespace with '\0' and update * the token array tokens with pointer to start of each token and length. * Returns total number of tokens. The last valid token is the terminal * token (value points to the first unprocessed character of the string and * length zero). * * Usage example: * * while(tokenize_command(command, ncommand, tokens, max_tokens) > 0) { * for(int ix = 0; tokens[ix].length != 0; ix++) { * ... * } * ncommand = tokens[ix].value - command; * command = tokens[ix].value; * } */ static size_t tokenize_command(char *command, token_t *tokens, const size_t max_tokens) { char *s, *e; size_t ntokens = 0; size_t len = strlen(command); unsigned int i = 0; assert(command != NULL && tokens != NULL && max_tokens > 1); s = e = command; for (i = 0; i < len; i++) { if (*e == ' ') { if (s != e) { tokens[ntokens].value = s; tokens[ntokens].length = e - s; ntokens++; *e = '\0'; if (ntokens == max_tokens - 1) { e++; s = e; /* so we don't add an extra token */ break; } } s = e + 1; } e++; } if (s != e) { tokens[ntokens].value = s; tokens[ntokens].length = e - s; ntokens++; } /* * If we scanned the whole string, the terminal value pointer is null, * otherwise it is the first unprocessed character. */ tokens[ntokens].value = *e == '\0' ? NULL : e; tokens[ntokens].length = 0; ntokens++; return ntokens; } /* set up a connection to write a buffer then free it, used for stats */ static void write_and_free(conn *c, char *buf, int bytes) { if (buf) { c->write_and_free = buf; c->wcurr = buf; c->wbytes = bytes; conn_set_state(c, conn_write); c->write_and_go = conn_new_cmd; } else { out_of_memory(c, "SERVER_ERROR out of memory writing stats"); } } static inline bool set_noreply_maybe(conn *c, token_t *tokens, size_t ntokens) { int noreply_index = ntokens - 2; /* NOTE: this function is not the first place where we are going to send the reply. We could send it instead from process_command() if the request line has wrong number of tokens. However parsing malformed line for "noreply" option is not reliable anyway, so it can't be helped. */ if (tokens[noreply_index].value && strcmp(tokens[noreply_index].value, "noreply") == 0) { c->noreply = true; } return c->noreply; } void append_stat(const char *name, ADD_STAT add_stats, conn *c, const char *fmt, ...) { char val_str[STAT_VAL_LEN]; int vlen; va_list ap; assert(name); assert(add_stats); assert(c); assert(fmt); va_start(ap, fmt); vlen = vsnprintf(val_str, sizeof(val_str) - 1, fmt, ap); va_end(ap); add_stats(name, strlen(name), val_str, vlen, c); } inline static void process_stats_detail(conn *c, const char *command) { assert(c != NULL); if (strcmp(command, "on") == 0) { settings.detail_enabled = 1; out_string(c, "OK"); } else if (strcmp(command, "off") == 0) { settings.detail_enabled = 0; out_string(c, "OK"); } else if (strcmp(command, "dump") == 0) { int len; char *stats = stats_prefix_dump(&len); write_and_free(c, stats, len); } else { out_string(c, "CLIENT_ERROR usage: stats detail on|off|dump"); } } /* return server specific stats only */ static void server_stats(ADD_STAT add_stats, conn *c) { pid_t pid = getpid(); rel_time_t now = current_time; struct thread_stats thread_stats; threadlocal_stats_aggregate(&thread_stats); struct slab_stats slab_stats; slab_stats_aggregate(&thread_stats, &slab_stats); #ifdef EXTSTORE struct extstore_stats st; #endif #ifndef WIN32 struct rusage usage; getrusage(RUSAGE_SELF, &usage); #endif /* !WIN32 */ STATS_LOCK(); APPEND_STAT("pid", "%lu", (long)pid); APPEND_STAT("uptime", "%u", now - ITEM_UPDATE_INTERVAL); APPEND_STAT("time", "%ld", now + (long)process_started); APPEND_STAT("version", "%s", VERSION); APPEND_STAT("libevent", "%s", event_get_version()); APPEND_STAT("pointer_size", "%d", (int)(8 * sizeof(void *))); #ifndef WIN32 append_stat("rusage_user", add_stats, c, "%ld.%06ld", (long)usage.ru_utime.tv_sec, (long)usage.ru_utime.tv_usec); append_stat("rusage_system", add_stats, c, "%ld.%06ld", (long)usage.ru_stime.tv_sec, (long)usage.ru_stime.tv_usec); #endif /* !WIN32 */ APPEND_STAT("max_connections", "%d", settings.maxconns); APPEND_STAT("curr_connections", "%llu", (unsigned long long)stats_state.curr_conns - 1); APPEND_STAT("total_connections", "%llu", (unsigned long long)stats.total_conns); if (settings.maxconns_fast) { APPEND_STAT("rejected_connections", "%llu", (unsigned long long)stats.rejected_conns); } APPEND_STAT("connection_structures", "%u", stats_state.conn_structs); APPEND_STAT("reserved_fds", "%u", stats_state.reserved_fds); APPEND_STAT("cmd_get", "%llu", (unsigned long long)thread_stats.get_cmds); APPEND_STAT("cmd_set", "%llu", (unsigned long long)slab_stats.set_cmds); APPEND_STAT("cmd_flush", "%llu", (unsigned long long)thread_stats.flush_cmds); APPEND_STAT("cmd_touch", "%llu", (unsigned long long)thread_stats.touch_cmds); APPEND_STAT("cmd_meta", "%llu", (unsigned long long)thread_stats.meta_cmds); APPEND_STAT("get_hits", "%llu", (unsigned long long)slab_stats.get_hits); APPEND_STAT("get_misses", "%llu", (unsigned long long)thread_stats.get_misses); APPEND_STAT("get_expired", "%llu", (unsigned long long)thread_stats.get_expired); APPEND_STAT("get_flushed", "%llu", (unsigned long long)thread_stats.get_flushed); #ifdef EXTSTORE if (c->thread->storage) { APPEND_STAT("get_extstore", "%llu", (unsigned long long)thread_stats.get_extstore); APPEND_STAT("get_aborted_extstore", "%llu", (unsigned long long)thread_stats.get_aborted_extstore); APPEND_STAT("get_oom_extstore", "%llu", (unsigned long long)thread_stats.get_oom_extstore); APPEND_STAT("recache_from_extstore", "%llu", (unsigned long long)thread_stats.recache_from_extstore); APPEND_STAT("miss_from_extstore", "%llu", (unsigned long long)thread_stats.miss_from_extstore); APPEND_STAT("badcrc_from_extstore", "%llu", (unsigned long long)thread_stats.badcrc_from_extstore); } #endif APPEND_STAT("delete_misses", "%llu", (unsigned long long)thread_stats.delete_misses); APPEND_STAT("delete_hits", "%llu", (unsigned long long)slab_stats.delete_hits); APPEND_STAT("incr_misses", "%llu", (unsigned long long)thread_stats.incr_misses); APPEND_STAT("incr_hits", "%llu", (unsigned long long)slab_stats.incr_hits); APPEND_STAT("decr_misses", "%llu", (unsigned long long)thread_stats.decr_misses); APPEND_STAT("decr_hits", "%llu", (unsigned long long)slab_stats.decr_hits); APPEND_STAT("cas_misses", "%llu", (unsigned long long)thread_stats.cas_misses); APPEND_STAT("cas_hits", "%llu", (unsigned long long)slab_stats.cas_hits); APPEND_STAT("cas_badval", "%llu", (unsigned long long)slab_stats.cas_badval); APPEND_STAT("touch_hits", "%llu", (unsigned long long)slab_stats.touch_hits); APPEND_STAT("touch_misses", "%llu", (unsigned long long)thread_stats.touch_misses); APPEND_STAT("auth_cmds", "%llu", (unsigned long long)thread_stats.auth_cmds); APPEND_STAT("auth_errors", "%llu", (unsigned long long)thread_stats.auth_errors); if (settings.idle_timeout) { APPEND_STAT("idle_kicks", "%llu", (unsigned long long)thread_stats.idle_kicks); } APPEND_STAT("bytes_read", "%llu", (unsigned long long)thread_stats.bytes_read); APPEND_STAT("bytes_written", "%llu", (unsigned long long)thread_stats.bytes_written); APPEND_STAT("limit_maxbytes", "%llu", (unsigned long long)settings.maxbytes); APPEND_STAT("accepting_conns", "%u", stats_state.accepting_conns); APPEND_STAT("listen_disabled_num", "%llu", (unsigned long long)stats.listen_disabled_num); APPEND_STAT("time_in_listen_disabled_us", "%llu", stats.time_in_listen_disabled_us); APPEND_STAT("threads", "%d", settings.num_threads); APPEND_STAT("conn_yields", "%llu", (unsigned long long)thread_stats.conn_yields); APPEND_STAT("hash_power_level", "%u", stats_state.hash_power_level); APPEND_STAT("hash_bytes", "%llu", (unsigned long long)stats_state.hash_bytes); APPEND_STAT("hash_is_expanding", "%u", stats_state.hash_is_expanding); if (settings.slab_reassign) { APPEND_STAT("slab_reassign_rescues", "%llu", stats.slab_reassign_rescues); APPEND_STAT("slab_reassign_chunk_rescues", "%llu", stats.slab_reassign_chunk_rescues); APPEND_STAT("slab_reassign_evictions_nomem", "%llu", stats.slab_reassign_evictions_nomem); APPEND_STAT("slab_reassign_inline_reclaim", "%llu", stats.slab_reassign_inline_reclaim); APPEND_STAT("slab_reassign_busy_items", "%llu", stats.slab_reassign_busy_items); APPEND_STAT("slab_reassign_busy_deletes", "%llu", stats.slab_reassign_busy_deletes); APPEND_STAT("slab_reassign_running", "%u", stats_state.slab_reassign_running); APPEND_STAT("slabs_moved", "%llu", stats.slabs_moved); } if (settings.lru_crawler) { APPEND_STAT("lru_crawler_running", "%u", stats_state.lru_crawler_running); APPEND_STAT("lru_crawler_starts", "%u", stats.lru_crawler_starts); } if (settings.lru_maintainer_thread) { APPEND_STAT("lru_maintainer_juggles", "%llu", (unsigned long long)stats.lru_maintainer_juggles); } APPEND_STAT("malloc_fails", "%llu", (unsigned long long)stats.malloc_fails); APPEND_STAT("log_worker_dropped", "%llu", (unsigned long long)stats.log_worker_dropped); APPEND_STAT("log_worker_written", "%llu", (unsigned long long)stats.log_worker_written); APPEND_STAT("log_watcher_skipped", "%llu", (unsigned long long)stats.log_watcher_skipped); APPEND_STAT("log_watcher_sent", "%llu", (unsigned long long)stats.log_watcher_sent); STATS_UNLOCK(); #ifdef EXTSTORE if (c->thread->storage) { STATS_LOCK(); APPEND_STAT("extstore_compact_lost", "%llu", (unsigned long long)stats.extstore_compact_lost); APPEND_STAT("extstore_compact_rescues", "%llu", (unsigned long long)stats.extstore_compact_rescues); APPEND_STAT("extstore_compact_skipped", "%llu", (unsigned long long)stats.extstore_compact_skipped); STATS_UNLOCK(); extstore_get_stats(c->thread->storage, &st); APPEND_STAT("extstore_page_allocs", "%llu", (unsigned long long)st.page_allocs); APPEND_STAT("extstore_page_evictions", "%llu", (unsigned long long)st.page_evictions); APPEND_STAT("extstore_page_reclaims", "%llu", (unsigned long long)st.page_reclaims); APPEND_STAT("extstore_pages_free", "%llu", (unsigned long long)st.pages_free); APPEND_STAT("extstore_pages_used", "%llu", (unsigned long long)st.pages_used); APPEND_STAT("extstore_objects_evicted", "%llu", (unsigned long long)st.objects_evicted); APPEND_STAT("extstore_objects_read", "%llu", (unsigned long long)st.objects_read); APPEND_STAT("extstore_objects_written", "%llu", (unsigned long long)st.objects_written); APPEND_STAT("extstore_objects_used", "%llu", (unsigned long long)st.objects_used); APPEND_STAT("extstore_bytes_evicted", "%llu", (unsigned long long)st.bytes_evicted); APPEND_STAT("extstore_bytes_written", "%llu", (unsigned long long)st.bytes_written); APPEND_STAT("extstore_bytes_read", "%llu", (unsigned long long)st.bytes_read); APPEND_STAT("extstore_bytes_used", "%llu", (unsigned long long)st.bytes_used); APPEND_STAT("extstore_bytes_fragmented", "%llu", (unsigned long long)st.bytes_fragmented); APPEND_STAT("extstore_limit_maxbytes", "%llu", (unsigned long long)(st.page_count * st.page_size)); APPEND_STAT("extstore_io_queue", "%llu", (unsigned long long)(st.io_queue)); } #endif #ifdef TLS if (settings.ssl_enabled) { APPEND_STAT("time_since_server_cert_refresh", "%u", now - settings.ssl_last_cert_refresh_time); } #endif } static void process_stat_settings(ADD_STAT add_stats, void *c) { assert(add_stats); APPEND_STAT("maxbytes", "%llu", (unsigned long long)settings.maxbytes); APPEND_STAT("maxconns", "%d", settings.maxconns); APPEND_STAT("tcpport", "%d", settings.port); APPEND_STAT("udpport", "%d", settings.udpport); APPEND_STAT("inter", "%s", settings.inter ? settings.inter : "NULL"); APPEND_STAT("verbosity", "%d", settings.verbose); APPEND_STAT("oldest", "%lu", (unsigned long)settings.oldest_live); APPEND_STAT("evictions", "%s", settings.evict_to_free ? "on" : "off"); APPEND_STAT("domain_socket", "%s", settings.socketpath ? settings.socketpath : "NULL"); APPEND_STAT("umask", "%o", settings.access); APPEND_STAT("growth_factor", "%.2f", settings.factor); APPEND_STAT("chunk_size", "%d", settings.chunk_size); APPEND_STAT("num_threads", "%d", settings.num_threads); APPEND_STAT("num_threads_per_udp", "%d", settings.num_threads_per_udp); APPEND_STAT("stat_key_prefix", "%c", settings.prefix_delimiter); APPEND_STAT("detail_enabled", "%s", settings.detail_enabled ? "yes" : "no"); APPEND_STAT("reqs_per_event", "%d", settings.reqs_per_event); APPEND_STAT("cas_enabled", "%s", settings.use_cas ? "yes" : "no"); APPEND_STAT("tcp_backlog", "%d", settings.backlog); APPEND_STAT("binding_protocol", "%s", prot_text(settings.binding_protocol)); APPEND_STAT("auth_enabled_sasl", "%s", settings.sasl ? "yes" : "no"); APPEND_STAT("auth_enabled_ascii", "%s", settings.auth_file ? settings.auth_file : "no"); APPEND_STAT("item_size_max", "%d", settings.item_size_max); APPEND_STAT("maxconns_fast", "%s", settings.maxconns_fast ? "yes" : "no"); APPEND_STAT("hashpower_init", "%d", settings.hashpower_init); APPEND_STAT("slab_reassign", "%s", settings.slab_reassign ? "yes" : "no"); APPEND_STAT("slab_automove", "%d", settings.slab_automove); APPEND_STAT("slab_automove_ratio", "%.2f", settings.slab_automove_ratio); APPEND_STAT("slab_automove_window", "%u", settings.slab_automove_window); APPEND_STAT("slab_chunk_max", "%d", settings.slab_chunk_size_max); APPEND_STAT("lru_crawler", "%s", settings.lru_crawler ? "yes" : "no"); APPEND_STAT("lru_crawler_sleep", "%d", settings.lru_crawler_sleep); APPEND_STAT("lru_crawler_tocrawl", "%lu", (unsigned long)settings.lru_crawler_tocrawl); APPEND_STAT("tail_repair_time", "%d", settings.tail_repair_time); APPEND_STAT("flush_enabled", "%s", settings.flush_enabled ? "yes" : "no"); APPEND_STAT("dump_enabled", "%s", settings.dump_enabled ? "yes" : "no"); APPEND_STAT("hash_algorithm", "%s", settings.hash_algorithm); APPEND_STAT("lru_maintainer_thread", "%s", settings.lru_maintainer_thread ? "yes" : "no"); APPEND_STAT("lru_segmented", "%s", settings.lru_segmented ? "yes" : "no"); APPEND_STAT("hot_lru_pct", "%d", settings.hot_lru_pct); APPEND_STAT("warm_lru_pct", "%d", settings.warm_lru_pct); APPEND_STAT("hot_max_factor", "%.2f", settings.hot_max_factor); APPEND_STAT("warm_max_factor", "%.2f", settings.warm_max_factor); APPEND_STAT("temp_lru", "%s", settings.temp_lru ? "yes" : "no"); APPEND_STAT("temporary_ttl", "%u", settings.temporary_ttl); APPEND_STAT("idle_timeout", "%d", settings.idle_timeout); APPEND_STAT("watcher_logbuf_size", "%u", settings.logger_watcher_buf_size); APPEND_STAT("worker_logbuf_size", "%u", settings.logger_buf_size); APPEND_STAT("track_sizes", "%s", item_stats_sizes_status() ? "yes" : "no"); APPEND_STAT("inline_ascii_response", "%s", "no"); // setting is dead, cannot be yes. #ifdef HAVE_DROP_PRIVILEGES APPEND_STAT("drop_privileges", "%s", settings.drop_privileges ? "yes" : "no"); #endif #ifdef EXTSTORE APPEND_STAT("ext_item_size", "%u", settings.ext_item_size); APPEND_STAT("ext_item_age", "%u", settings.ext_item_age); APPEND_STAT("ext_low_ttl", "%u", settings.ext_low_ttl); APPEND_STAT("ext_recache_rate", "%u", settings.ext_recache_rate); APPEND_STAT("ext_wbuf_size", "%u", settings.ext_wbuf_size); APPEND_STAT("ext_compact_under", "%u", settings.ext_compact_under); APPEND_STAT("ext_drop_under", "%u", settings.ext_drop_under); APPEND_STAT("ext_max_frag", "%.2f", settings.ext_max_frag); APPEND_STAT("slab_automove_freeratio", "%.3f", settings.slab_automove_freeratio); APPEND_STAT("ext_drop_unread", "%s", settings.ext_drop_unread ? "yes" : "no"); #endif #ifdef TLS APPEND_STAT("ssl_enabled", "%s", settings.ssl_enabled ? "yes" : "no"); APPEND_STAT("ssl_chain_cert", "%s", settings.ssl_chain_cert); APPEND_STAT("ssl_key", "%s", settings.ssl_key); APPEND_STAT("ssl_verify_mode", "%d", settings.ssl_verify_mode); APPEND_STAT("ssl_keyformat", "%d", settings.ssl_keyformat); APPEND_STAT("ssl_ciphers", "%s", settings.ssl_ciphers ? settings.ssl_ciphers : "NULL"); APPEND_STAT("ssl_ca_cert", "%s", settings.ssl_ca_cert ? settings.ssl_ca_cert : "NULL"); APPEND_STAT("ssl_wbuf_size", "%u", settings.ssl_wbuf_size); #endif } static int nz_strcmp(int nzlength, const char *nz, const char *z) { int zlength=strlen(z); return (zlength == nzlength) && (strncmp(nz, z, zlength) == 0) ? 0 : -1; } static bool get_stats(const char *stat_type, int nkey, ADD_STAT add_stats, void *c) { bool ret = true; if (add_stats != NULL) { if (!stat_type) { /* prepare general statistics for the engine */ STATS_LOCK(); APPEND_STAT("bytes", "%llu", (unsigned long long)stats_state.curr_bytes); APPEND_STAT("curr_items", "%llu", (unsigned long long)stats_state.curr_items); APPEND_STAT("total_items", "%llu", (unsigned long long)stats.total_items); STATS_UNLOCK(); APPEND_STAT("slab_global_page_pool", "%u", global_page_pool_size(NULL)); item_stats_totals(add_stats, c); } else if (nz_strcmp(nkey, stat_type, "items") == 0) { item_stats(add_stats, c); } else if (nz_strcmp(nkey, stat_type, "slabs") == 0) { slabs_stats(add_stats, c); } else if (nz_strcmp(nkey, stat_type, "sizes") == 0) { item_stats_sizes(add_stats, c); } else if (nz_strcmp(nkey, stat_type, "sizes_enable") == 0) { item_stats_sizes_enable(add_stats, c); } else if (nz_strcmp(nkey, stat_type, "sizes_disable") == 0) { item_stats_sizes_disable(add_stats, c); } else { ret = false; } } else { ret = false; } return ret; } static inline void get_conn_text(const conn *c, const int af, char* addr, struct sockaddr *sock_addr) { char addr_text[MAXPATHLEN]; addr_text[0] = '\0'; const char *protoname = "?"; unsigned short port = 0; size_t pathlen = 0; switch (af) { case AF_INET: (void) inet_ntop(af, &((struct sockaddr_in *)sock_addr)->sin_addr, addr_text, sizeof(addr_text) - 1); port = ntohs(((struct sockaddr_in *)sock_addr)->sin_port); protoname = IS_UDP(c->transport) ? "udp" : "tcp"; break; case AF_INET6: addr_text[0] = '['; addr_text[1] = '\0'; if (inet_ntop(af, &((struct sockaddr_in6 *)sock_addr)->sin6_addr, addr_text + 1, sizeof(addr_text) - 2)) { strcat(addr_text, "]"); } port = ntohs(((struct sockaddr_in6 *)sock_addr)->sin6_port); protoname = IS_UDP(c->transport) ? "udp6" : "tcp6"; break; case AF_UNIX: // this strncpy call originally could piss off an address // sanitizer; we supplied the size of the dest buf as a limiter, // but optimized versions of strncpy could read past the end of // *src while looking for a null terminator. Since buf and // sun_path here are both on the stack they could even overlap, // which is "undefined". In all OSS versions of strncpy I could // find this has no effect; it'll still only copy until the first null // terminator is found. Thus it's possible to get the OS to // examine past the end of sun_path but it's unclear to me if this // can cause any actual problem. // // We need a safe_strncpy util function but I'll punt on figuring // that out for now. pathlen = sizeof(((struct sockaddr_un *)sock_addr)->sun_path); if (MAXPATHLEN <= pathlen) { pathlen = MAXPATHLEN - 1; } strncpy(addr_text, ((struct sockaddr_un *)sock_addr)->sun_path, pathlen); addr_text[pathlen] = '\0'; protoname = "unix"; break; } if (strlen(addr_text) < 2) { /* Most likely this is a connected UNIX-domain client which * has no peer socket address, but there's no portable way * to tell for sure. */ sprintf(addr_text, "", af); } if (port) { sprintf(addr, "%s:%s:%u", protoname, addr_text, port); } else { sprintf(addr, "%s:%s", protoname, addr_text); } } static void conn_to_str(const conn *c, char *addr, char *svr_addr) { if (!c) { strcpy(addr, ""); } else if (c->state == conn_closed) { strcpy(addr, ""); } else { struct sockaddr_in6 local_addr; struct sockaddr *sock_addr = (void *)&c->request_addr; /* For listen ports and idle UDP ports, show listen address */ if (c->state == conn_listening || (IS_UDP(c->transport) && c->state == conn_read)) { socklen_t local_addr_len = sizeof(local_addr); if (getsockname(c->sfd, (struct sockaddr *)&local_addr, &local_addr_len) == 0) { sock_addr = (struct sockaddr *)&local_addr; } } get_conn_text(c, sock_addr->sa_family, addr, sock_addr); if (c->state != conn_listening && !(IS_UDP(c->transport) && c->state == conn_read)) { struct sockaddr_storage svr_sock_addr; socklen_t svr_addr_len = sizeof(svr_sock_addr); getsockname(c->sfd, (struct sockaddr *)&svr_sock_addr, &svr_addr_len); get_conn_text(c, svr_sock_addr.ss_family, svr_addr, (struct sockaddr *)&svr_sock_addr); } } } static void process_stats_conns(ADD_STAT add_stats, void *c) { int i; char key_str[STAT_KEY_LEN]; char val_str[STAT_VAL_LEN]; size_t extras_len = sizeof("unix:") + sizeof("65535"); char addr[MAXPATHLEN + extras_len]; char svr_addr[MAXPATHLEN + extras_len]; int klen = 0, vlen = 0; assert(add_stats); for (i = 0; i < max_fds; i++) { if (conns[i]) { /* This is safe to do unlocked because conns are never freed; the * worst that'll happen will be a minor inconsistency in the * output -- not worth the complexity of the locking that'd be * required to prevent it. */ if (IS_UDP(conns[i]->transport)) { APPEND_NUM_STAT(i, "UDP", "%s", "UDP"); } if (conns[i]->state != conn_closed) { conn_to_str(conns[i], addr, svr_addr); APPEND_NUM_STAT(i, "addr", "%s", addr); if (conns[i]->state != conn_listening && !(IS_UDP(conns[i]->transport) && conns[i]->state == conn_read)) { APPEND_NUM_STAT(i, "listen_addr", "%s", svr_addr); } APPEND_NUM_STAT(i, "state", "%s", state_text(conns[i]->state)); APPEND_NUM_STAT(i, "secs_since_last_cmd", "%d", current_time - conns[i]->last_cmd_time); } } } } #ifdef EXTSTORE static void process_extstore_stats(ADD_STAT add_stats, conn *c) { int i; char key_str[STAT_KEY_LEN]; char val_str[STAT_VAL_LEN]; int klen = 0, vlen = 0; struct extstore_stats st; assert(add_stats); void *storage = c->thread->storage; extstore_get_stats(storage, &st); st.page_data = calloc(st.page_count, sizeof(struct extstore_page_data)); extstore_get_page_data(storage, &st); for (i = 0; i < st.page_count; i++) { APPEND_NUM_STAT(i, "version", "%llu", (unsigned long long) st.page_data[i].version); APPEND_NUM_STAT(i, "bytes", "%llu", (unsigned long long) st.page_data[i].bytes_used); APPEND_NUM_STAT(i, "bucket", "%u", st.page_data[i].bucket); APPEND_NUM_STAT(i, "free_bucket", "%u", st.page_data[i].free_bucket); } } #endif static void process_stat(conn *c, token_t *tokens, const size_t ntokens) { const char *subcommand = tokens[SUBCOMMAND_TOKEN].value; assert(c != NULL); if (ntokens < 2) { out_string(c, "CLIENT_ERROR bad command line"); return; } if (ntokens == 2) { server_stats(&append_stats, c); (void)get_stats(NULL, 0, &append_stats, c); } else if (strcmp(subcommand, "reset") == 0) { stats_reset(); out_string(c, "RESET"); return; } else if (strcmp(subcommand, "detail") == 0) { /* NOTE: how to tackle detail with binary? */ if (ntokens < 4) process_stats_detail(c, ""); /* outputs the error message */ else process_stats_detail(c, tokens[2].value); /* Output already generated */ return; } else if (strcmp(subcommand, "settings") == 0) { process_stat_settings(&append_stats, c); } else if (strcmp(subcommand, "cachedump") == 0) { char *buf; unsigned int bytes, id, limit = 0; if (!settings.dump_enabled) { out_string(c, "CLIENT_ERROR stats cachedump not allowed"); return; } if (ntokens < 5) { out_string(c, "CLIENT_ERROR bad command line"); return; } if (!safe_strtoul(tokens[2].value, &id) || !safe_strtoul(tokens[3].value, &limit)) { out_string(c, "CLIENT_ERROR bad command line format"); return; } if (id >= MAX_NUMBER_OF_SLAB_CLASSES) { out_string(c, "CLIENT_ERROR Illegal slab id"); return; } buf = item_cachedump(id, limit, &bytes); write_and_free(c, buf, bytes); return; } else if (strcmp(subcommand, "conns") == 0) { process_stats_conns(&append_stats, c); #ifdef EXTSTORE } else if (strcmp(subcommand, "extstore") == 0) { process_extstore_stats(&append_stats, c); #endif } else { /* getting here means that the subcommand is either engine specific or is invalid. query the engine and see. */ if (get_stats(subcommand, strlen(subcommand), &append_stats, c)) { if (c->stats.buffer == NULL) { out_of_memory(c, "SERVER_ERROR out of memory writing stats"); } else { write_and_free(c, c->stats.buffer, c->stats.offset); c->stats.buffer = NULL; } } else { out_string(c, "ERROR"); } return; } /* append terminator and start the transfer */ append_stats(NULL, 0, NULL, 0, c); if (c->stats.buffer == NULL) { out_of_memory(c, "SERVER_ERROR out of memory writing stats"); } else { write_and_free(c, c->stats.buffer, c->stats.offset); c->stats.buffer = NULL; } } /* client flags == 0 means use no storage for client flags */ static inline int make_ascii_get_suffix(char *suffix, item *it, bool return_cas, int nbytes) { char *p = suffix; *p = ' '; p++; if (FLAGS_SIZE(it) == 0) { *p = '0'; p++; } else { p = itoa_u32(*((uint32_t *) ITEM_suffix(it)), p); } *p = ' '; p = itoa_u32(nbytes-2, p+1); if (return_cas) { *p = ' '; p = itoa_u64(ITEM_get_cas(it), p+1); } *p = '\r'; *(p+1) = '\n'; *(p+2) = '\0'; return (p - suffix) + 2; } #define IT_REFCOUNT_LIMIT 60000 static inline item* limited_get(char *key, size_t nkey, conn *c, uint32_t exptime, bool should_touch, bool do_update) { item *it; if (should_touch) { it = item_touch(key, nkey, exptime, c); } else { it = item_get(key, nkey, c, do_update); } if (it && it->refcount > IT_REFCOUNT_LIMIT) { item_remove(it); it = NULL; } return it; } // Semantics are different than limited_get; since the item is returned // locked, caller can directly change what it needs. // though it might eventually be a better interface to sink it all into // items.c. static inline item* limited_get_locked(char *key, size_t nkey, conn *c, bool do_update, uint32_t *hv) { item *it; it = item_get_locked(key, nkey, c, do_update, hv); if (it && it->refcount > IT_REFCOUNT_LIMIT) { do_item_remove(it); it = NULL; item_unlock(*hv); } return it; } static inline int _ascii_get_expand_ilist(conn *c, int i) { if (i >= c->isize) { item **new_list = realloc(c->ilist, sizeof(item *) * c->isize * 2); if (new_list) { c->isize *= 2; c->ilist = new_list; } else { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); return -1; } } return 0; } static inline char *_ascii_get_suffix_buf(conn *c, int i) { char *suffix; /* Goofy mid-flight realloc. */ if (i >= c->suffixsize) { char **new_suffix_list = realloc(c->suffixlist, sizeof(char *) * c->suffixsize * 2); if (new_suffix_list) { c->suffixsize *= 2; c->suffixlist = new_suffix_list; } else { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); return NULL; } } suffix = do_cache_alloc(c->thread->suffix_cache); if (suffix == NULL) { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); out_of_memory(c, "SERVER_ERROR out of memory making CAS suffix"); return NULL; } *(c->suffixlist + i) = suffix; return suffix; } #ifdef EXTSTORE // FIXME: This runs in the IO thread. to get better IO performance this should // simply mark the io wrapper with the return value and decrement wrapleft, if // zero redispatching. Still a bit of work being done in the side thread but // minimized at least. static void _get_extstore_cb(void *e, obj_io *io, int ret) { // FIXME: assumes success io_wrap *wrap = (io_wrap *)io->data; conn *c = wrap->c; assert(wrap->active == true); item *read_it = (item *)io->buf; bool miss = false; // TODO: How to do counters for hit/misses? if (ret < 1) { miss = true; } else { uint32_t crc2; uint32_t crc = (uint32_t) read_it->exptime; int x; // item is chunked, crc the iov's if (io->iov != NULL) { // first iov is the header, which we don't use beyond crc crc2 = crc32c(0, (char *)io->iov[0].iov_base+STORE_OFFSET, io->iov[0].iov_len-STORE_OFFSET); // make sure it's not sent. hack :( io->iov[0].iov_len = 0; for (x = 1; x < io->iovcnt; x++) { crc2 = crc32c(crc2, (char *)io->iov[x].iov_base, io->iov[x].iov_len); } } else { crc2 = crc32c(0, (char *)read_it+STORE_OFFSET, io->len-STORE_OFFSET); } if (crc != crc2) { miss = true; wrap->badcrc = true; } } if (miss) { int i; struct iovec *v; // TODO: This should be movable to the worker thread. if (c->protocol == binary_prot) { protocol_binary_response_header *header = (protocol_binary_response_header *)c->wbuf; // this zeroes out the iovecs since binprot never stacks them. if (header->response.keylen) { write_bin_miss_response(c, ITEM_key(wrap->hdr_it), wrap->hdr_it->nkey); } else { write_bin_miss_response(c, 0, 0); } } else { for (i = 0; i < wrap->iovec_count; i++) { v = &c->iov[wrap->iovec_start + i]; v->iov_len = 0; v->iov_base = NULL; } } wrap->miss = true; } else { assert(read_it->slabs_clsid != 0); // kill \r\n for binprot if (io->iov == NULL) { c->iov[wrap->iovec_data].iov_base = ITEM_data(read_it); if (c->protocol == binary_prot) c->iov[wrap->iovec_data].iov_len -= 2; } else { // FIXME: Might need to go back and ensure chunked binprots don't // ever span two chunks for the final \r\n if (c->protocol == binary_prot) { if (io->iov[io->iovcnt-1].iov_len >= 2) { io->iov[io->iovcnt-1].iov_len -= 2; } else { io->iov[io->iovcnt-1].iov_len = 0; io->iov[io->iovcnt-2].iov_len -= 1; } } } wrap->miss = false; // iov_len is already set // TODO: Should do that here instead and cuddle in the wrap object } c->io_wrapleft--; wrap->active = false; //assert(c->io_wrapleft >= 0); // All IO's have returned, lets re-attach this connection to our original // thread. if (c->io_wrapleft == 0) { assert(c->io_queued == true); c->io_queued = false; redispatch_conn(c); } } // FIXME: This completely breaks UDP support. static inline int _get_extstore(conn *c, item *it, int iovst, int iovcnt) { #ifdef NEED_ALIGN item_hdr hdr; memcpy(&hdr, ITEM_data(it), sizeof(hdr)); #else item_hdr *hdr = (item_hdr *)ITEM_data(it); #endif size_t ntotal = ITEM_ntotal(it); unsigned int clsid = slabs_clsid(ntotal); item *new_it; bool chunked = false; if (ntotal > settings.slab_chunk_size_max) { // Pull a chunked item header. uint32_t flags; FLAGS_CONV(it, flags); new_it = item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, it->nbytes); assert(new_it == NULL || (new_it->it_flags & ITEM_CHUNKED)); chunked = true; } else { new_it = do_item_alloc_pull(ntotal, clsid); } if (new_it == NULL) return -1; assert(!c->io_queued); // FIXME: debugging. // so we can free the chunk on a miss new_it->slabs_clsid = clsid; io_wrap *io = do_cache_alloc(c->thread->io_cache); io->active = true; io->miss = false; io->badcrc = false; // io_wrap owns the reference for this object now. io->hdr_it = it; // FIXME: error handling. // The offsets we'll wipe on a miss. io->iovec_start = iovst; io->iovec_count = iovcnt; // This is probably super dangerous. keep it at 0 and fill into wrap // object? if (chunked) { unsigned int ciovcnt = 1; size_t remain = new_it->nbytes; item_chunk *chunk = (item_chunk *) ITEM_schunk(new_it); io->io.iov = &c->iov[c->iovused]; // fill the header so we can get the full data + crc back. add_iov(c, new_it, ITEM_ntotal(new_it) - new_it->nbytes); while (remain > 0) { chunk = do_item_alloc_chunk(chunk, remain); if (chunk == NULL) { item_remove(new_it); do_cache_free(c->thread->io_cache, io); return -1; } add_iov(c, chunk->data, (remain < chunk->size) ? remain : chunk->size); chunk->used = (remain < chunk->size) ? remain : chunk->size; remain -= chunk->size; ciovcnt++; } io->io.iovcnt = ciovcnt; // header object was already accounted for, remove one from total io->iovec_count += ciovcnt-1; } else { io->io.iov = NULL; io->iovec_data = c->iovused; add_iov(c, "", it->nbytes); } io->io.buf = (void *)new_it; // The offset we'll fill in on a hit. io->c = c; // We need to stack the sub-struct IO's together as well. if (c->io_wraplist) { io->io.next = &c->io_wraplist->io; } else { io->io.next = NULL; } // IO queue for this connection. io->next = c->io_wraplist; c->io_wraplist = io; assert(c->io_wrapleft >= 0); c->io_wrapleft++; // reference ourselves for the callback. io->io.data = (void *)io; // Now, fill in io->io based on what was in our header. #ifdef NEED_ALIGN io->io.page_version = hdr.page_version; io->io.page_id = hdr.page_id; io->io.offset = hdr.offset; #else io->io.page_version = hdr->page_version; io->io.page_id = hdr->page_id; io->io.offset = hdr->offset; #endif io->io.len = ntotal; io->io.mode = OBJ_IO_READ; io->io.cb = _get_extstore_cb; //fprintf(stderr, "EXTSTORE: IO stacked %u\n", io->iovec_data); // FIXME: This stat needs to move to reflect # of flash hits vs misses // for now it's a good gauge on how often we request out to flash at // least. pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.get_extstore++; pthread_mutex_unlock(&c->thread->stats.mutex); return 0; } #endif /* ntokens is overwritten here... shrug.. */ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, bool return_cas, bool should_touch) { char *key; size_t nkey; int i = 0; int si = 0; item *it; token_t *key_token = &tokens[KEY_TOKEN]; char *suffix; int32_t exptime_int = 0; rel_time_t exptime = 0; bool fail_length = false; assert(c != NULL); if (should_touch) { // For get and touch commands, use first token as exptime if (!safe_strtol(tokens[1].value, &exptime_int)) { out_string(c, "CLIENT_ERROR invalid exptime argument"); return; } key_token++; exptime = realtime(exptime_int); } do { while(key_token->length != 0) { key = key_token->value; nkey = key_token->length; if (nkey > KEY_MAX_LENGTH) { fail_length = true; goto stop; } it = limited_get(key, nkey, c, exptime, should_touch, DO_UPDATE); if (settings.detail_enabled) { stats_prefix_record_get(key, nkey, NULL != it); } if (it) { if (_ascii_get_expand_ilist(c, i) != 0) { item_remove(it); goto stop; } /* * Construct the response. Each hit adds three elements to the * outgoing data list: * "VALUE " * key * " " + flags + " " + data length + "\r\n" + data (with \r\n) */ { MEMCACHED_COMMAND_GET(c->sfd, ITEM_key(it), it->nkey, it->nbytes, ITEM_get_cas(it)); int nbytes; suffix = _ascii_get_suffix_buf(c, si); if (suffix == NULL) { item_remove(it); goto stop; } si++; nbytes = it->nbytes; int suffix_len = make_ascii_get_suffix(suffix, it, return_cas, nbytes); if (add_iov(c, "VALUE ", 6) != 0 || add_iov(c, ITEM_key(it), it->nkey) != 0 || add_iov(c, suffix, suffix_len) != 0) { item_remove(it); goto stop; } #ifdef EXTSTORE if (it->it_flags & ITEM_HDR) { if (_get_extstore(c, it, c->iovused-3, 4) != 0) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.get_oom_extstore++; pthread_mutex_unlock(&c->thread->stats.mutex); item_remove(it); goto stop; } } else if ((it->it_flags & ITEM_CHUNKED) == 0) { #else if ((it->it_flags & ITEM_CHUNKED) == 0) { #endif add_iov(c, ITEM_data(it), it->nbytes); } else if (add_chunked_item_iovs(c, it, it->nbytes) != 0) { item_remove(it); goto stop; } } if (settings.verbose > 1) { int ii; fprintf(stderr, ">%d sending key ", c->sfd); for (ii = 0; ii < it->nkey; ++ii) { fprintf(stderr, "%c", key[ii]); } fprintf(stderr, "\n"); } /* item_get() has incremented it->refcount for us */ pthread_mutex_lock(&c->thread->stats.mutex); if (should_touch) { c->thread->stats.touch_cmds++; c->thread->stats.slab_stats[ITEM_clsid(it)].touch_hits++; } else { c->thread->stats.lru_hits[it->slabs_clsid]++; c->thread->stats.get_cmds++; } pthread_mutex_unlock(&c->thread->stats.mutex); #ifdef EXTSTORE /* If ITEM_HDR, an io_wrap owns the reference. */ if ((it->it_flags & ITEM_HDR) == 0) { *(c->ilist + i) = it; i++; } #else *(c->ilist + i) = it; i++; #endif } else { pthread_mutex_lock(&c->thread->stats.mutex); if (should_touch) { c->thread->stats.touch_cmds++; c->thread->stats.touch_misses++; } else { c->thread->stats.get_misses++; c->thread->stats.get_cmds++; } MEMCACHED_COMMAND_GET(c->sfd, key, nkey, -1, 0); pthread_mutex_unlock(&c->thread->stats.mutex); } key_token++; } /* * If the command string hasn't been fully processed, get the next set * of tokens. */ if(key_token->value != NULL) { ntokens = tokenize_command(key_token->value, tokens, MAX_TOKENS); key_token = tokens; } } while(key_token->value != NULL); stop: c->icurr = c->ilist; c->ileft = i; c->suffixcurr = c->suffixlist; c->suffixleft = si; if (settings.verbose > 1) fprintf(stderr, ">%d END\n", c->sfd); /* If the loop was terminated because of out-of-memory, it is not reliable to add END\r\n to the buffer, because it might not end in \r\n. So we send SERVER_ERROR instead. */ if (key_token->value != NULL || add_iov(c, "END\r\n", 5) != 0 || (IS_UDP(c->transport) && build_udp_headers(c) != 0)) { if (fail_length) { out_string(c, "CLIENT_ERROR bad command line format"); } else { out_of_memory(c, "SERVER_ERROR out of memory writing get response"); } conn_release_items(c); } else { conn_set_state(c, conn_mwrite); c->msgcurr = 0; } } // slow snprintf for debugging purposes. static void process_meta_command(conn *c, token_t *tokens, const size_t ntokens) { assert(c != NULL); if (tokens[KEY_TOKEN].length > KEY_MAX_LENGTH) { out_string(c, "CLIENT_ERROR bad command line format"); return; } char *key = tokens[KEY_TOKEN].value; size_t nkey = tokens[KEY_TOKEN].length; item *it = limited_get(key, nkey, c, 0, false, DONT_UPDATE); if (it) { size_t total = 0; size_t ret; // similar to out_string(). memcpy(c->wbuf, "ME ", 3); total += 3; memcpy(c->wbuf + total, ITEM_key(it), it->nkey); total += it->nkey; c->wbuf[total] = ' '; total++; ret = snprintf(c->wbuf + total, c->wsize - (it->nkey + 12), "exp=%d la=%llu cas=%llu fetch=%s cls=%u size=%lu\r\nEN\r\n", (it->exptime == 0) ? -1 : (current_time - it->exptime), (unsigned long long)(current_time - it->time), (unsigned long long)ITEM_get_cas(it), (it->it_flags & ITEM_FETCHED) ? "yes" : "no", ITEM_clsid(it), (unsigned long) ITEM_ntotal(it)); item_remove(it); c->wbytes = total + ret; c->wcurr = c->wbuf; conn_set_state(c, conn_write); c->write_and_go = conn_new_cmd; } else { out_string(c, "EN"); } pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.meta_cmds++; pthread_mutex_unlock(&c->thread->stats.mutex); } #define MFLAG_MAX_OPT_LENGTH 20 #define MFLAG_MAX_OPAQUE_LENGTH 32 struct _meta_flags { unsigned int no_update :1; unsigned int locked :1; unsigned int vivify :1; unsigned int la :1; unsigned int hit :1; unsigned int value :1; unsigned int set_stale :1; unsigned int no_reply :1; }; static int _meta_flag_preparse(char *opts, size_t olen, struct _meta_flags *of) { unsigned int i; // NOTE: 'seen' is one of those need-to-microbench situation to do this via bit vector or // simple 8 bit array; need two divs and a shift vs a bit more memory. // Leave it simple for now, optimize later. uint8_t seen[127] = {0}; // also count how many tokens should be necessary to parse. int tokens = 0; for (i = 0; i < olen; i++) { uint8_t o = (uint8_t)opts[i]; // zero out repeat flags so we don't over-parse for return data. if (o >= 127 || seen[o] != 0) { return -1; } seen[o] = 1; // FIXME: alphabetize. switch (opts[i]) { case 'N': of->locked = 1; of->vivify = 1; tokens++; break; case 'T': tokens++; of->locked = 1; break; case 'R': of->locked = 1; tokens++; break; case 'l': of->la = 1; of->locked = 1; // need locked to delay LRU bump break; case 'O': tokens++; break; case 'k': // known but no special handling case 's': case 't': case 'c': case 'v': case 'f': break; case 'h': of->locked = 1; // need locked to delay LRU bump break; case 'u': of->no_update = 1; break; case 'q': of->no_reply = 1; break; // mset-related. case 'F': case 'S': case 'C': tokens++; break; case 'I': of->set_stale = 1; break; default: // unknown flag, bail. return -1; } } return tokens; } static void process_mget_command(conn *c, token_t *tokens, const size_t ntokens) { char *key; size_t nkey; item *it; char *opts; char *fp = NULL; char *p = c->wbuf; size_t olen; unsigned int i = 0; int32_t rtokens = 0; // remaining tokens available. struct _meta_flags of = {0}; // option bitflags. uint32_t hv; // cached hash value for unlocking an item. bool failed = false; bool item_created = false; bool won_token = false; bool ttl_set = false; char *errstr; assert(c != NULL); if (tokens[KEY_TOKEN].length > KEY_MAX_LENGTH) { out_errstring(c, "CLIENT_ERROR bad command line format"); return; } key = tokens[KEY_TOKEN].value; nkey = tokens[KEY_TOKEN].length; // NOTE: final token has length == 0. // KEY_TOKEN == 1. 0 is command. rtokens = 3; // cmd, key, final. if (ntokens == 3) { // Default flag options. Might not be the best idea. opts = "sftv"; olen = 4; } else { // need to parse out the options. opts = tokens[KEY_TOKEN + 1].value; olen = tokens[KEY_TOKEN + 1].length; rtokens++; } if (olen > MFLAG_MAX_OPT_LENGTH) { out_errstring(c, "CLIENT_ERROR options flags are too long"); return; } // Copy opts into wbuf and advance pointer. // We return the initial options + extra indicator flags. // we reserve 4 bytes in front of the buffer, for up to three extra flags // we can tag plus the initial space. // This could be simpler by adding two iov's for the header line, but this // is a hot path so trying to keep those to a minimum. fp = c->wbuf + 6; memcpy(fp, opts, olen); p = fp + olen; fp--; // next token, or final space, goes here. // scrubs duplicated options and sets flags for how to load the item. int mfres = _meta_flag_preparse(opts, olen, &of); if (mfres + rtokens != ntokens) { if (mfres == -1) { out_errstring(c, "CLIENT_ERROR invalid or duplicate flag"); } else { out_errstring(c, "CLIENT_ERROR incorrect number of tokens supplied"); } return; } rtokens = KEY_TOKEN + 2; c->noreply = of.no_reply; // TODO: need to indicate if the item was overflowed or not? // I think we do, since an overflow shouldn't trigger an alloc/replace. if (!of.locked) { it = limited_get(key, nkey, c, 0, false, !of.no_update); } else { // If we had to lock the item, we're doing our own bump later. it = limited_get_locked(key, nkey, c, DONT_UPDATE, &hv); } if (it == NULL && of.vivify) { // Fill in the exptime during parsing later. it = item_alloc(key, nkey, 0, realtime(0), 2); // We don't actually need any of do_store_item's logic: // - already fetched and missed an existing item. // - lock is still held. // - not append/prepend/replace // - not testing CAS if (it != NULL) { // I look forward to the day I get rid of this :) memcpy(ITEM_data(it), "\r\n", 2); // NOTE: This initializes the CAS value. do_item_link(it, hv); item_created = true; } } // don't have to check result of add_iov() since the iov size defaults are // enough. if (it) { int32_t exptime_int = 0; // Has this item already sent a token? // Important to do this here so we don't send W with Z. // Isn't critical, but easier for client authors to understand. if (it->it_flags & ITEM_TOKEN_SENT) { *fp = 'Z'; fp--; } if (it->it_flags & ITEM_STALE) { *fp = 'X'; fp--; // FIXME: think hard about this. is this a default, or a flag? if ((it->it_flags & ITEM_TOKEN_SENT) == 0) { // If we're stale but no token already sent, now send one. won_token = true; } } for (i = 0; i < olen; i++) { switch (opts[i]) { case 'T': ttl_set = true; if (!safe_strtol(tokens[rtokens].value, &exptime_int)) { errstr = "CLIENT_ERROR bad tokens in command line format"; goto error; } // FIXME: check for < 0, or stoul and cast here. it->exptime = realtime(exptime_int); rtokens++; break; case 'N': if (item_created) { if (!safe_strtol(tokens[rtokens].value, &exptime_int)) { errstr = "CLIENT_ERROR bad tokens in command line format"; goto error; } // FIXME: check for < 0, or stoul and cast here. it->exptime = realtime(exptime_int); won_token = true; } rtokens++; // always consume the token. break; case 'R': // If we haven't autovivified and supplied token is less // than current TTL, mark a win. if ((it->it_flags & ITEM_TOKEN_SENT) == 0 && !item_created && it->exptime != 0) { if (!safe_strtol(tokens[rtokens].value, &exptime_int)) { errstr = "CLIENT_ERROR bad tokens in command line format"; goto error; } if (it->exptime - current_time < exptime_int) { won_token = true; } } rtokens++; break; case 's': *p = ' '; p = itoa_u32(it->nbytes-2, p+1); break; case 't': // TODO: ensure this is correct for autoviv case. // or, I guess users can put N before t? // TTL remaining as of this request. // needs to be relative because server clocks may not be in sync. *p = ' '; if (it->exptime == 0) { *(p+1) = '-'; *(p+2) = '1'; p += 3; } else { p = itoa_u32(it->exptime - current_time, p+1); } break; case 'c': *p = ' '; p = itoa_u64(ITEM_get_cas(it), p+1); break; case 'v': of.value = 1; break; case 'f': *p = ' '; if (FLAGS_SIZE(it) == 0) { *(p+1) = '0'; p += 2; } else { p = itoa_u32(*((uint32_t *) ITEM_suffix(it)), p+1); } break; case 'l': *p = ' '; p = itoa_u32(current_time - it->time, p+1); break; case 'h': *p = ' '; if (it->it_flags & ITEM_FETCHED) { *(p+1) = '1'; } else { *(p+1) = '0'; } p += 2; break; case 'O': if (tokens[rtokens].length > MFLAG_MAX_OPAQUE_LENGTH) { errstr = "CLIENT_ERROR opaque token too long"; goto error; } *p = ' '; memcpy(p+1, tokens[rtokens].value, tokens[rtokens].length); p += tokens[rtokens].length + 1; rtokens++; break; case 'k': *p = ' '; memcpy(p+1, ITEM_key(it), it->nkey); p += it->nkey + 1; break; } } if (won_token) { // Mark a win into the flag buffer. *fp = 'W'; fp--; // walk backwards for next token. it->it_flags |= ITEM_TOKEN_SENT; } // TODO: Benchmark unlocking here vs later. _get_extstore() could be // intensive so probably better to unlock here after we're done // fiddling with the item header. *p = '\r'; *(p+1) = '\n'; *(p+2) = '\0'; p += 2; // tag initial space to the front of the buffer, ahead of any extra // flags that were added. *fp = ' '; *(fp-1) = 'A'; *(fp-2) = 'V'; fp -= 2; // finally, chain in the buffer. // fp includes the flags. add_iov(c, fp, p - fp); if (of.value) { #ifdef EXTSTORE if (it->it_flags & ITEM_HDR) { // this bizarre interface is instructing _get_extstore() to // "walk back and zero out" this many iovec's on an internal // miss. kills the VALUE + key + header stitched above. int iovcnt = 2; int iovst = c->iovused - 1; if (_get_extstore(c, it, iovst, iovcnt) != 0) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.get_oom_extstore++; pthread_mutex_unlock(&c->thread->stats.mutex); failed = true; } } else if ((it->it_flags & ITEM_CHUNKED) == 0) { add_iov(c, ITEM_data(it), it->nbytes); } else { add_chunked_item_iovs(c, it, it->nbytes); } #else if ((it->it_flags & ITEM_CHUNKED) == 0) { add_iov(c, ITEM_data(it), it->nbytes); } else { add_chunked_item_iovs(c, it, it->nbytes); } #endif } if (!c->noreply) { add_iov(c, "EN\r\n", 4); } // need to hold the ref at least because of the key above. #ifdef EXTSTORE if (!failed) { if ((it->it_flags & ITEM_HDR) != 0 && of.value) { // Only have extstore clean if header and returning value. c->item = NULL; } else { c->item = it; } } else { if (of.locked) { do_item_remove(it); } else { item_remove(it); } } #else c->item = it; #endif } else { failed = true; } if (of.locked) { // Delayed bump so we could get fetched/last access time/etc. // TODO: before segmented LRU, last-access time would only update // every 60 seconds. Currently it doesn't update at all if an item is // marked as ACTIVE. I believe this is a bug... in segmented mode // there's no reason to avoid bumping la on every access. if (!of.no_update && it != NULL) { do_item_bump(c, it, hv); } item_unlock(hv); } // we count this command as a normal one if we've gotten this far. // TODO: for autovivify case, miss never happens. Is this okay? if (!failed) { pthread_mutex_lock(&c->thread->stats.mutex); if (ttl_set) { c->thread->stats.touch_cmds++; c->thread->stats.slab_stats[ITEM_clsid(it)].touch_hits++; } else { c->thread->stats.lru_hits[it->slabs_clsid]++; c->thread->stats.get_cmds++; } pthread_mutex_unlock(&c->thread->stats.mutex); conn_set_state(c, conn_write); c->write_and_go = conn_new_cmd; } else { pthread_mutex_lock(&c->thread->stats.mutex); if (ttl_set) { c->thread->stats.touch_cmds++; c->thread->stats.touch_misses++; } else { c->thread->stats.get_misses++; c->thread->stats.get_cmds++; } MEMCACHED_COMMAND_GET(c->sfd, key, nkey, -1, 0); pthread_mutex_unlock(&c->thread->stats.mutex); // This gets elided in noreply mode. out_string(c, "EN"); } return; error: if (it && of.locked) { item_unlock(hv); } out_errstring(c, errstr); } static void process_mset_command(conn *c, token_t *tokens, const size_t ntokens) { char *key; size_t nkey; unsigned int flags = 0; uint32_t exptime_int = 0; uint64_t req_cas_id = 0; char *p; time_t exptime; int vlen = 0; int rtokens; item *it; char *opts; int olen, i; short comm = NREAD_SET; struct _meta_flags of = {0}; // option bitflags. char *errstr = "CLIENT_ERROR bad command line format"; uint32_t hv; assert(c != NULL); // TODO: most of this is identical to mget. if (tokens[KEY_TOKEN].length > KEY_MAX_LENGTH) { out_errstring(c, "CLIENT_ERROR bad command line format"); return; } key = tokens[KEY_TOKEN].value; nkey = tokens[KEY_TOKEN].length; rtokens = 3; // cmd, key, final. if (ntokens == 3) { out_errstring(c, "CLIENT_ERROR bad command line format"); return; } opts = tokens[KEY_TOKEN + 1].value; olen = tokens[KEY_TOKEN + 1].length; rtokens++; if (olen > MFLAG_MAX_OPT_LENGTH) { out_errstring(c, "CLIENT_ERROR options flags too long"); return; } // copy opts before munging them. // first two chars is status code. p = c->wbuf + 3; memcpy(p, opts, olen); p += olen; // scrubs duplicated options and sets flags for how to load the item. // TODO: I, E, APL? int mfres = _meta_flag_preparse(opts, olen, &of); if (mfres + rtokens != ntokens) { if (mfres == -1) { out_errstring(c, "CLIENT_ERROR invalid or duplicate flag"); } else { out_errstring(c, "CLIENT_ERROR incorrect number of tokens supplied"); } return; } // Set noreply after tokens are understood. c->noreply = of.no_reply; rtokens = KEY_TOKEN + 2; for (i = 0; i < olen; i++) { switch (opts[i]) { case 'F': if (!safe_strtoul(tokens[rtokens].value, &flags)) { goto error; } rtokens++; break; case 'C': if (!safe_strtoull(tokens[rtokens].value, &req_cas_id)) { goto error; } comm = NREAD_CAS; rtokens++; break; case 'S': if (!safe_strtol(tokens[rtokens].value, &vlen)) { goto error; } rtokens++; break; case 'T': if (!safe_strtoul(tokens[rtokens].value, &exptime_int)) { goto error; } rtokens++; break; // TODO: macro perhaps? case 'O': if (tokens[rtokens].length > MFLAG_MAX_OPAQUE_LENGTH) { errstr = "CLIENT_ERROR opaque token too long"; goto error; } *p = ' '; memcpy(p+1, tokens[rtokens].value, tokens[rtokens].length); p += tokens[rtokens].length + 1; rtokens++; break; case 'k': *p = ' '; memcpy(p+1, key, nkey); p += nkey + 1; break; } } /* Ubuntu 8.04 breaks when I pass exptime to safe_strtol */ exptime = exptime_int; /* Negative exptimes can underflow and end up immortal. realtime() will immediately expire values that are greater than REALTIME_MAXDELTA, but less than process_started, so lets aim for that. */ if (exptime < 0) exptime = REALTIME_MAXDELTA + 1; // TODO: can we treat vlen as unsigned? :( if (vlen < 0 || vlen > (INT_MAX - 2)) { // TODO: specific error. goto error; } vlen += 2; it = item_alloc(key, nkey, flags, realtime(exptime), vlen); if (it == 0) { enum store_item_type status; // TODO: These could be normalized codes (TL and OM). Need to // reorganize the output stuff a bit though. if (! item_size_ok(nkey, flags, vlen)) { errstr = "SERVER_ERROR object too large for cache"; status = TOO_LARGE; } else { errstr = "SERVER_ERROR out of memory storing object"; status = NO_MEMORY; } // FIXME: LOGGER_LOG specific to mset, include options. LOGGER_LOG(c->thread->l, LOG_MUTATIONS, LOGGER_ITEM_STORE, NULL, status, comm, key, nkey, 0, 0); /* Avoid stale data persisting in cache because we failed alloc. */ it = item_get_locked(key, nkey, c, DONT_UPDATE, &hv); if (it) { do_item_unlink(it, hv); STORAGE_delete(c->thread->storage, it); do_item_remove(it); } item_unlock(hv); goto error; } ITEM_set_cas(it, req_cas_id); c->item = it; #ifdef NEED_ALIGN if (it->it_flags & ITEM_CHUNKED) { c->ritem = ITEM_schunk(it); } else { c->ritem = ITEM_data(it); } #else c->ritem = ITEM_data(it); #endif c->rlbytes = it->nbytes; // TODO: Could support other modes (append/prepend/replace/add) c->cmd = comm; if (of.set_stale && comm == NREAD_CAS) { c->set_stale = true; } // We've written the status line into wbuf, use wbytes to finalize later. c->wbytes = p - c->wbuf; c->mset_res = true; conn_set_state(c, conn_nread); return; error: /* swallow the data line */ c->write_and_go = conn_swallow; c->sbytes = vlen; // Note: no errors possible after the item was successfully allocated. // So we're just looking at dumping error codes and returning. out_errstring(c, errstr); } static void process_mdelete_command(conn *c, token_t *tokens, const size_t ntokens) { char *key; size_t nkey; uint32_t exptime_int = 0; uint64_t req_cas_id = 0; char *p; int rtokens; item *it = NULL; char *opts; int olen, i; uint32_t hv; struct _meta_flags of = {0}; // option bitflags. char *errstr = "CLIENT_ERROR bad command line format"; assert(c != NULL); // TODO: most of this is identical to mget. if (tokens[KEY_TOKEN].length > KEY_MAX_LENGTH) { out_string(c, "CLIENT_ERROR bad command line format"); return; } key = tokens[KEY_TOKEN].value; nkey = tokens[KEY_TOKEN].length; rtokens = 3; // cmd, key, final. // rtokens == 0 acts like a normal immediate delete if (ntokens == 3) { opts = ""; olen = 0; } else { opts = tokens[KEY_TOKEN + 1].value; olen = tokens[KEY_TOKEN + 1].length; rtokens++; } if (olen > MFLAG_MAX_OPT_LENGTH) { out_string(c, "CLIENT_ERROR options flags too long"); return; } // copy opts before munging them. // first two chars is status code. p = c->wbuf + 3; memcpy(p, opts, olen); p += olen; // scrubs duplicated options and sets flags for how to load the item. int mfres = _meta_flag_preparse(opts, olen, &of); if (mfres + rtokens != ntokens) { if (mfres == -1) { out_errstring(c, "CLIENT_ERROR invalid or duplicate flag"); } else { out_errstring(c, "CLIENT_ERROR incorrect number of tokens supplied"); } return; } rtokens = KEY_TOKEN + 2; c->noreply = of.no_reply; assert(c != NULL); bool new_ttl = false; for (i = 0; i < olen; i++) { switch (opts[i]) { case 'C': if (!safe_strtoull(tokens[rtokens].value, &req_cas_id)) { goto error; } rtokens++; break; case 'T': if (!safe_strtoul(tokens[rtokens].value, &exptime_int)) { goto error; } new_ttl = true; rtokens++; break; // TODO: macro perhaps? case 'O': if (tokens[rtokens].length > MFLAG_MAX_OPAQUE_LENGTH) { errstr = "CLIENT_ERROR opaque token too long"; goto error; } *p = ' '; memcpy(p+1, tokens[rtokens].value, tokens[rtokens].length); p += tokens[rtokens].length + 1; rtokens++; break; case 'k': *p = ' '; memcpy(p+1, key, nkey); p += nkey + 1; rtokens++; break; } } it = item_get_locked(key, nkey, c, DONT_UPDATE, &hv); if (it) { MEMCACHED_COMMAND_DELETE(c->sfd, ITEM_key(it), it->nkey); // allow only deleting/marking if a CAS value matches. if (req_cas_id != 0 && ITEM_get_cas(it) != req_cas_id) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.delete_misses++; pthread_mutex_unlock(&c->thread->stats.mutex); // Need to ensure client gets this response. memcpy(c->wbuf, "EX ", 3); c->wbytes = p - c->wbuf; out_mstring(c, DISABLE_NOREPLY); goto cleanup; } // If we're to set this item as stale, we don't actually want to // delete it. We mark the stale bit, bump CAS, and update exptime if // we were supplied a new TTL. if (of.set_stale) { if (new_ttl) { it->exptime = realtime(exptime_int); } it->it_flags |= ITEM_STALE; // Also need to remove TOKEN_SENT, so next client can win. it->it_flags &= ~ITEM_TOKEN_SENT; ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0); memcpy(c->wbuf, "DE ", 3); // TODO: just use two diff funcs? :| // also, take len for arg? c->wbytes = p - c->wbuf; out_mstring(c, ALLOW_NOREPLY); } else { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.slab_stats[ITEM_clsid(it)].delete_hits++; pthread_mutex_unlock(&c->thread->stats.mutex); do_item_unlink(it, hv); STORAGE_delete(c->thread->storage, it); memcpy(c->wbuf, "DE ", 3); c->wbytes = p - c->wbuf; out_mstring(c, ALLOW_NOREPLY); } goto cleanup; } else { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.delete_misses++; pthread_mutex_unlock(&c->thread->stats.mutex); memcpy(c->wbuf, "NF ", 3); c->wbytes = p - c->wbuf; out_mstring(c, ALLOW_NOREPLY); goto cleanup; } cleanup: if (it) { do_item_remove(it); } // Item is always returned locked, even if missing. item_unlock(hv); return; error: out_errstring(c, errstr); } static void process_update_command(conn *c, token_t *tokens, const size_t ntokens, int comm, bool handle_cas) { char *key; size_t nkey; unsigned int flags; int32_t exptime_int = 0; time_t exptime; int vlen; uint64_t req_cas_id=0; item *it; assert(c != NULL); set_noreply_maybe(c, tokens, ntokens); if (tokens[KEY_TOKEN].length > KEY_MAX_LENGTH) { out_string(c, "CLIENT_ERROR bad command line format"); return; } key = tokens[KEY_TOKEN].value; nkey = tokens[KEY_TOKEN].length; if (! (safe_strtoul(tokens[2].value, (uint32_t *)&flags) && safe_strtol(tokens[3].value, &exptime_int) && safe_strtol(tokens[4].value, (int32_t *)&vlen))) { out_string(c, "CLIENT_ERROR bad command line format"); return; } /* Ubuntu 8.04 breaks when I pass exptime to safe_strtol */ exptime = exptime_int; /* Negative exptimes can underflow and end up immortal. realtime() will immediately expire values that are greater than REALTIME_MAXDELTA, but less than process_started, so lets aim for that. */ if (exptime < 0) exptime = REALTIME_MAXDELTA + 1; // does cas value exist? if (handle_cas) { if (!safe_strtoull(tokens[5].value, &req_cas_id)) { out_string(c, "CLIENT_ERROR bad command line format"); return; } } if (vlen < 0 || vlen > (INT_MAX - 2)) { out_string(c, "CLIENT_ERROR bad command line format"); return; } vlen += 2; if (settings.detail_enabled) { stats_prefix_record_set(key, nkey); } it = item_alloc(key, nkey, flags, realtime(exptime), vlen); if (it == 0) { enum store_item_type status; if (! item_size_ok(nkey, flags, vlen)) { out_string(c, "SERVER_ERROR object too large for cache"); status = TOO_LARGE; } else { out_of_memory(c, "SERVER_ERROR out of memory storing object"); status = NO_MEMORY; } LOGGER_LOG(c->thread->l, LOG_MUTATIONS, LOGGER_ITEM_STORE, NULL, status, comm, key, nkey, 0, 0, c->sfd); /* swallow the data line */ c->write_and_go = conn_swallow; c->sbytes = vlen; /* Avoid stale data persisting in cache because we failed alloc. * Unacceptable for SET. Anywhere else too? */ if (comm == NREAD_SET) { it = item_get(key, nkey, c, DONT_UPDATE); if (it) { item_unlink(it); STORAGE_delete(c->thread->storage, it); item_remove(it); } } return; } ITEM_set_cas(it, req_cas_id); c->item = it; #ifdef NEED_ALIGN if (it->it_flags & ITEM_CHUNKED) { c->ritem = ITEM_schunk(it); } else { c->ritem = ITEM_data(it); } #else c->ritem = ITEM_data(it); #endif c->rlbytes = it->nbytes; c->cmd = comm; conn_set_state(c, conn_nread); } static void process_touch_command(conn *c, token_t *tokens, const size_t ntokens) { char *key; size_t nkey; int32_t exptime_int = 0; item *it; assert(c != NULL); set_noreply_maybe(c, tokens, ntokens); if (tokens[KEY_TOKEN].length > KEY_MAX_LENGTH) { out_string(c, "CLIENT_ERROR bad command line format"); return; } key = tokens[KEY_TOKEN].value; nkey = tokens[KEY_TOKEN].length; if (!safe_strtol(tokens[2].value, &exptime_int)) { out_string(c, "CLIENT_ERROR invalid exptime argument"); return; } it = item_touch(key, nkey, realtime(exptime_int), c); if (it) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.touch_cmds++; c->thread->stats.slab_stats[ITEM_clsid(it)].touch_hits++; pthread_mutex_unlock(&c->thread->stats.mutex); out_string(c, "TOUCHED"); item_remove(it); } else { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.touch_cmds++; c->thread->stats.touch_misses++; pthread_mutex_unlock(&c->thread->stats.mutex); out_string(c, "NOT_FOUND"); } } static void process_arithmetic_command(conn *c, token_t *tokens, const size_t ntokens, const bool incr) { char temp[INCR_MAX_STORAGE_LEN]; uint64_t delta; char *key; size_t nkey; assert(c != NULL); set_noreply_maybe(c, tokens, ntokens); if (tokens[KEY_TOKEN].length > KEY_MAX_LENGTH) { out_string(c, "CLIENT_ERROR bad command line format"); return; } key = tokens[KEY_TOKEN].value; nkey = tokens[KEY_TOKEN].length; if (!safe_strtoull(tokens[2].value, &delta)) { out_string(c, "CLIENT_ERROR invalid numeric delta argument"); return; } switch(add_delta(c, key, nkey, incr, delta, temp, NULL)) { case OK: out_string(c, temp); break; case NON_NUMERIC: out_string(c, "CLIENT_ERROR cannot increment or decrement non-numeric value"); break; case EOM: out_of_memory(c, "SERVER_ERROR out of memory"); break; case DELTA_ITEM_NOT_FOUND: pthread_mutex_lock(&c->thread->stats.mutex); if (incr) { c->thread->stats.incr_misses++; } else { c->thread->stats.decr_misses++; } pthread_mutex_unlock(&c->thread->stats.mutex); out_string(c, "NOT_FOUND"); break; case DELTA_ITEM_CAS_MISMATCH: break; /* Should never get here */ } } /* * adds a delta value to a numeric item. * * c connection requesting the operation * it item to adjust * incr true to increment value, false to decrement * delta amount to adjust value by * buf buffer for response string * * returns a response string to send back to the client. */ enum delta_result_type do_add_delta(conn *c, const char *key, const size_t nkey, const bool incr, const int64_t delta, char *buf, uint64_t *cas, const uint32_t hv) { char *ptr; uint64_t value; int res; item *it; it = do_item_get(key, nkey, hv, c, DONT_UPDATE); if (!it) { return DELTA_ITEM_NOT_FOUND; } /* Can't delta zero byte values. 2-byte are the "\r\n" */ /* Also can't delta for chunked items. Too large to be a number */ #ifdef EXTSTORE if (it->nbytes <= 2 || (it->it_flags & (ITEM_CHUNKED|ITEM_HDR)) != 0) { #else if (it->nbytes <= 2 || (it->it_flags & (ITEM_CHUNKED)) != 0) { #endif do_item_remove(it); return NON_NUMERIC; } if (cas != NULL && *cas != 0 && ITEM_get_cas(it) != *cas) { do_item_remove(it); return DELTA_ITEM_CAS_MISMATCH; } ptr = ITEM_data(it); if (!safe_strtoull(ptr, &value)) { do_item_remove(it); return NON_NUMERIC; } if (incr) { value += delta; MEMCACHED_COMMAND_INCR(c->sfd, ITEM_key(it), it->nkey, value); } else { if(delta > value) { value = 0; } else { value -= delta; } MEMCACHED_COMMAND_DECR(c->sfd, ITEM_key(it), it->nkey, value); } pthread_mutex_lock(&c->thread->stats.mutex); if (incr) { c->thread->stats.slab_stats[ITEM_clsid(it)].incr_hits++; } else { c->thread->stats.slab_stats[ITEM_clsid(it)].decr_hits++; } pthread_mutex_unlock(&c->thread->stats.mutex); itoa_u64(value, buf); res = strlen(buf); /* refcount == 2 means we are the only ones holding the item, and it is * linked. We hold the item's lock in this function, so refcount cannot * increase. */ if (res + 2 <= it->nbytes && it->refcount == 2) { /* replace in-place */ /* When changing the value without replacing the item, we need to update the CAS on the existing item. */ /* We also need to fiddle it in the sizes tracker in case the tracking * was enabled at runtime, since it relies on the CAS value to know * whether to remove an item or not. */ item_stats_sizes_remove(it); ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0); item_stats_sizes_add(it); memcpy(ITEM_data(it), buf, res); memset(ITEM_data(it) + res, ' ', it->nbytes - res - 2); do_item_update(it); } else if (it->refcount > 1) { item *new_it; uint32_t flags; FLAGS_CONV(it, flags); new_it = do_item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, res + 2); if (new_it == 0) { do_item_remove(it); return EOM; } memcpy(ITEM_data(new_it), buf, res); memcpy(ITEM_data(new_it) + res, "\r\n", 2); item_replace(it, new_it, hv); // Overwrite the older item's CAS with our new CAS since we're // returning the CAS of the old item below. ITEM_set_cas(it, (settings.use_cas) ? ITEM_get_cas(new_it) : 0); do_item_remove(new_it); /* release our reference */ } else { /* Should never get here. This means we somehow fetched an unlinked * item. TODO: Add a counter? */ if (settings.verbose) { fprintf(stderr, "Tried to do incr/decr on invalid item\n"); } if (it->refcount == 1) do_item_remove(it); return DELTA_ITEM_NOT_FOUND; } if (cas) { *cas = ITEM_get_cas(it); /* swap the incoming CAS value */ } do_item_remove(it); /* release our reference */ return OK; } static void process_delete_command(conn *c, token_t *tokens, const size_t ntokens) { char *key; size_t nkey; item *it; uint32_t hv; assert(c != NULL); if (ntokens > 3) { bool hold_is_zero = strcmp(tokens[KEY_TOKEN+1].value, "0") == 0; bool sets_noreply = set_noreply_maybe(c, tokens, ntokens); bool valid = (ntokens == 4 && (hold_is_zero || sets_noreply)) || (ntokens == 5 && hold_is_zero && sets_noreply); if (!valid) { out_string(c, "CLIENT_ERROR bad command line format. " "Usage: delete [noreply]"); return; } } key = tokens[KEY_TOKEN].value; nkey = tokens[KEY_TOKEN].length; if(nkey > KEY_MAX_LENGTH) { out_string(c, "CLIENT_ERROR bad command line format"); return; } if (settings.detail_enabled) { stats_prefix_record_delete(key, nkey); } it = item_get_locked(key, nkey, c, DONT_UPDATE, &hv); if (it) { MEMCACHED_COMMAND_DELETE(c->sfd, ITEM_key(it), it->nkey); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.slab_stats[ITEM_clsid(it)].delete_hits++; pthread_mutex_unlock(&c->thread->stats.mutex); do_item_unlink(it, hv); STORAGE_delete(c->thread->storage, it); do_item_remove(it); /* release our reference */ out_string(c, "DELETED"); } else { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.delete_misses++; pthread_mutex_unlock(&c->thread->stats.mutex); out_string(c, "NOT_FOUND"); } item_unlock(hv); } static void process_verbosity_command(conn *c, token_t *tokens, const size_t ntokens) { unsigned int level; assert(c != NULL); set_noreply_maybe(c, tokens, ntokens); level = strtoul(tokens[1].value, NULL, 10); settings.verbose = level > MAX_VERBOSITY_LEVEL ? MAX_VERBOSITY_LEVEL : level; out_string(c, "OK"); return; } #ifdef MEMCACHED_DEBUG static void process_misbehave_command(conn *c) { int allowed = 0; // try opening new TCP socket int i = socket(AF_INET, SOCK_STREAM, 0); if (i != -1) { allowed++; close(i); } // try executing new commands i = system("sleep 0"); if (i != -1) { allowed++; } if (allowed) { out_string(c, "ERROR"); } else { out_string(c, "OK"); } } #endif static void process_slabs_automove_command(conn *c, token_t *tokens, const size_t ntokens) { unsigned int level; double ratio; assert(c != NULL); set_noreply_maybe(c, tokens, ntokens); if (strcmp(tokens[2].value, "ratio") == 0) { if (ntokens < 5 || !safe_strtod(tokens[3].value, &ratio)) { out_string(c, "ERROR"); return; } settings.slab_automove_ratio = ratio; } else { level = strtoul(tokens[2].value, NULL, 10); if (level == 0) { settings.slab_automove = 0; } else if (level == 1 || level == 2) { settings.slab_automove = level; } else { out_string(c, "ERROR"); return; } } out_string(c, "OK"); return; } /* TODO: decide on syntax for sampling? */ static void process_watch_command(conn *c, token_t *tokens, const size_t ntokens) { uint16_t f = 0; int x; assert(c != NULL); set_noreply_maybe(c, tokens, ntokens); if (!settings.watch_enabled) { out_string(c, "CLIENT_ERROR watch commands not allowed"); return; } if (ntokens > 2) { for (x = COMMAND_TOKEN + 1; x < ntokens - 1; x++) { if ((strcmp(tokens[x].value, "rawcmds") == 0)) { f |= LOG_RAWCMDS; } else if ((strcmp(tokens[x].value, "evictions") == 0)) { f |= LOG_EVICTIONS; } else if ((strcmp(tokens[x].value, "fetchers") == 0)) { f |= LOG_FETCHERS; } else if ((strcmp(tokens[x].value, "mutations") == 0)) { f |= LOG_MUTATIONS; } else if ((strcmp(tokens[x].value, "sysevents") == 0)) { f |= LOG_SYSEVENTS; } else { out_string(c, "ERROR"); return; } } } else { f |= LOG_FETCHERS; } switch(logger_add_watcher(c, c->sfd, f)) { case LOGGER_ADD_WATCHER_TOO_MANY: out_string(c, "WATCHER_TOO_MANY log watcher limit reached"); break; case LOGGER_ADD_WATCHER_FAILED: out_string(c, "WATCHER_FAILED failed to add log watcher"); break; case LOGGER_ADD_WATCHER_OK: conn_set_state(c, conn_watch); event_del(&c->event); break; } } static void process_memlimit_command(conn *c, token_t *tokens, const size_t ntokens) { uint32_t memlimit; assert(c != NULL); set_noreply_maybe(c, tokens, ntokens); if (!safe_strtoul(tokens[1].value, &memlimit)) { out_string(c, "ERROR"); } else { if (memlimit < 8) { out_string(c, "MEMLIMIT_TOO_SMALL cannot set maxbytes to less than 8m"); } else { if (memlimit > 1000000000) { out_string(c, "MEMLIMIT_ADJUST_FAILED input value is megabytes not bytes"); } else if (slabs_adjust_mem_limit((size_t) memlimit * 1024 * 1024)) { if (settings.verbose > 0) { fprintf(stderr, "maxbytes adjusted to %llum\n", (unsigned long long)memlimit); } out_string(c, "OK"); } else { out_string(c, "MEMLIMIT_ADJUST_FAILED out of bounds or unable to adjust"); } } } } static void process_lru_command(conn *c, token_t *tokens, const size_t ntokens) { uint32_t pct_hot; uint32_t pct_warm; double hot_factor; int32_t ttl; double factor; set_noreply_maybe(c, tokens, ntokens); if (strcmp(tokens[1].value, "tune") == 0 && ntokens >= 7) { if (!safe_strtoul(tokens[2].value, &pct_hot) || !safe_strtoul(tokens[3].value, &pct_warm) || !safe_strtod(tokens[4].value, &hot_factor) || !safe_strtod(tokens[5].value, &factor)) { out_string(c, "ERROR"); } else { if (pct_hot + pct_warm > 80) { out_string(c, "ERROR hot and warm pcts must not exceed 80"); } else if (factor <= 0 || hot_factor <= 0) { out_string(c, "ERROR hot/warm age factors must be greater than 0"); } else { settings.hot_lru_pct = pct_hot; settings.warm_lru_pct = pct_warm; settings.hot_max_factor = hot_factor; settings.warm_max_factor = factor; out_string(c, "OK"); } } } else if (strcmp(tokens[1].value, "mode") == 0 && ntokens >= 4 && settings.lru_maintainer_thread) { if (strcmp(tokens[2].value, "flat") == 0) { settings.lru_segmented = false; out_string(c, "OK"); } else if (strcmp(tokens[2].value, "segmented") == 0) { settings.lru_segmented = true; out_string(c, "OK"); } else { out_string(c, "ERROR"); } } else if (strcmp(tokens[1].value, "temp_ttl") == 0 && ntokens >= 4 && settings.lru_maintainer_thread) { if (!safe_strtol(tokens[2].value, &ttl)) { out_string(c, "ERROR"); } else { if (ttl < 0) { settings.temp_lru = false; } else { settings.temp_lru = true; settings.temporary_ttl = ttl; } out_string(c, "OK"); } } else { out_string(c, "ERROR"); } } #ifdef EXTSTORE static void process_extstore_command(conn *c, token_t *tokens, const size_t ntokens) { set_noreply_maybe(c, tokens, ntokens); bool ok = true; if (ntokens < 4) { ok = false; } else if (strcmp(tokens[1].value, "free_memchunks") == 0 && ntokens > 4) { /* per-slab-class free chunk setting. */ unsigned int clsid = 0; unsigned int limit = 0; if (!safe_strtoul(tokens[2].value, &clsid) || !safe_strtoul(tokens[3].value, &limit)) { ok = false; } else { if (clsid < MAX_NUMBER_OF_SLAB_CLASSES) { settings.ext_free_memchunks[clsid] = limit; } else { ok = false; } } } else if (strcmp(tokens[1].value, "item_size") == 0) { if (!safe_strtoul(tokens[2].value, &settings.ext_item_size)) ok = false; } else if (strcmp(tokens[1].value, "item_age") == 0) { if (!safe_strtoul(tokens[2].value, &settings.ext_item_age)) ok = false; } else if (strcmp(tokens[1].value, "low_ttl") == 0) { if (!safe_strtoul(tokens[2].value, &settings.ext_low_ttl)) ok = false; } else if (strcmp(tokens[1].value, "recache_rate") == 0) { if (!safe_strtoul(tokens[2].value, &settings.ext_recache_rate)) ok = false; } else if (strcmp(tokens[1].value, "compact_under") == 0) { if (!safe_strtoul(tokens[2].value, &settings.ext_compact_under)) ok = false; } else if (strcmp(tokens[1].value, "drop_under") == 0) { if (!safe_strtoul(tokens[2].value, &settings.ext_drop_under)) ok = false; } else if (strcmp(tokens[1].value, "max_frag") == 0) { if (!safe_strtod(tokens[2].value, &settings.ext_max_frag)) ok = false; } else if (strcmp(tokens[1].value, "drop_unread") == 0) { unsigned int v; if (!safe_strtoul(tokens[2].value, &v)) { ok = false; } else { settings.ext_drop_unread = v == 0 ? false : true; } } else { ok = false; } if (!ok) { out_string(c, "ERROR"); } else { out_string(c, "OK"); } } #endif static void process_command(conn *c, char *command) { token_t tokens[MAX_TOKENS]; size_t ntokens; int comm; assert(c != NULL); MEMCACHED_PROCESS_COMMAND_START(c->sfd, c->rcurr, c->rbytes); if (settings.verbose > 1) fprintf(stderr, "<%d %s\n", c->sfd, command); /* * for commands set/add/replace, we build an item and read the data * directly into it, then continue in nread_complete(). */ c->msgcurr = 0; c->msgused = 0; c->iovused = 0; if (add_msghdr(c) != 0) { out_of_memory(c, "SERVER_ERROR out of memory preparing response"); return; } ntokens = tokenize_command(command, tokens, MAX_TOKENS); if (ntokens >= 3 && ((strcmp(tokens[COMMAND_TOKEN].value, "get") == 0) || (strcmp(tokens[COMMAND_TOKEN].value, "bget") == 0))) { process_get_command(c, tokens, ntokens, false, false); } else if ((ntokens == 6 || ntokens == 7) && ((strcmp(tokens[COMMAND_TOKEN].value, "add") == 0 && (comm = NREAD_ADD)) || (strcmp(tokens[COMMAND_TOKEN].value, "set") == 0 && (comm = NREAD_SET)) || (strcmp(tokens[COMMAND_TOKEN].value, "replace") == 0 && (comm = NREAD_REPLACE)) || (strcmp(tokens[COMMAND_TOKEN].value, "prepend") == 0 && (comm = NREAD_PREPEND)) || (strcmp(tokens[COMMAND_TOKEN].value, "append") == 0 && (comm = NREAD_APPEND)) )) { process_update_command(c, tokens, ntokens, comm, false); } else if ((ntokens == 7 || ntokens == 8) && (strcmp(tokens[COMMAND_TOKEN].value, "cas") == 0 && (comm = NREAD_CAS))) { process_update_command(c, tokens, ntokens, comm, true); } else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "incr") == 0)) { process_arithmetic_command(c, tokens, ntokens, 1); } else if (ntokens >= 3 && (strcmp(tokens[COMMAND_TOKEN].value, "gets") == 0)) { process_get_command(c, tokens, ntokens, true, false); } else if (ntokens >= 3 && (strcmp(tokens[COMMAND_TOKEN].value, "mg") == 0)) { process_mget_command(c, tokens, ntokens); } else if (ntokens >= 3 && (strcmp(tokens[COMMAND_TOKEN].value, "ms") == 0)) { process_mset_command(c, tokens, ntokens); } else if (ntokens >= 3 && (strcmp(tokens[COMMAND_TOKEN].value, "md") == 0)) { process_mdelete_command(c, tokens, ntokens); } else if (ntokens >= 2 && (strcmp(tokens[COMMAND_TOKEN].value, "mn") == 0)) { out_string(c, "EN"); return; } else if (ntokens >= 2 && (strcmp(tokens[COMMAND_TOKEN].value, "me") == 0)) { process_meta_command(c, tokens, ntokens); return; } else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "decr") == 0)) { process_arithmetic_command(c, tokens, ntokens, 0); } else if (ntokens >= 3 && ntokens <= 5 && (strcmp(tokens[COMMAND_TOKEN].value, "delete") == 0)) { process_delete_command(c, tokens, ntokens); } else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "touch") == 0)) { process_touch_command(c, tokens, ntokens); } else if (ntokens >= 4 && (strcmp(tokens[COMMAND_TOKEN].value, "gat") == 0)) { process_get_command(c, tokens, ntokens, false, true); } else if (ntokens >= 4 && (strcmp(tokens[COMMAND_TOKEN].value, "gats") == 0)) { process_get_command(c, tokens, ntokens, true, true); } else if (ntokens >= 2 && (strcmp(tokens[COMMAND_TOKEN].value, "stats") == 0)) { process_stat(c, tokens, ntokens); } else if (ntokens >= 2 && ntokens <= 4 && (strcmp(tokens[COMMAND_TOKEN].value, "flush_all") == 0)) { time_t exptime = 0; rel_time_t new_oldest = 0; set_noreply_maybe(c, tokens, ntokens); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.flush_cmds++; pthread_mutex_unlock(&c->thread->stats.mutex); if (!settings.flush_enabled) { // flush_all is not allowed but we log it on stats out_string(c, "CLIENT_ERROR flush_all not allowed"); return; } if (ntokens != (c->noreply ? 3 : 2)) { exptime = strtol(tokens[1].value, NULL, 10); if(errno == ERANGE) { out_string(c, "CLIENT_ERROR bad command line format"); return; } } /* If exptime is zero realtime() would return zero too, and realtime(exptime) - 1 would overflow to the max unsigned value. So we process exptime == 0 the same way we do when no delay is given at all. */ if (exptime > 0) { new_oldest = realtime(exptime); } else { /* exptime == 0 */ new_oldest = current_time; } if (settings.use_cas) { settings.oldest_live = new_oldest - 1; if (settings.oldest_live <= current_time) settings.oldest_cas = get_cas_id(); } else { settings.oldest_live = new_oldest; } out_string(c, "OK"); return; } else if (ntokens == 2 && (strcmp(tokens[COMMAND_TOKEN].value, "version") == 0)) { out_string(c, "VERSION " VERSION); } else if (ntokens == 2 && (strcmp(tokens[COMMAND_TOKEN].value, "quit") == 0)) { conn_set_state(c, conn_closing); } else if (ntokens == 2 && (strcmp(tokens[COMMAND_TOKEN].value, "shutdown") == 0)) { if (settings.shutdown_command) { conn_set_state(c, conn_closing); raise(SIGINT); } else { out_string(c, "ERROR: shutdown not enabled"); } } else if (ntokens > 1 && strcmp(tokens[COMMAND_TOKEN].value, "slabs") == 0) { if (ntokens == 5 && strcmp(tokens[COMMAND_TOKEN + 1].value, "reassign") == 0) { int src, dst, rv; if (settings.slab_reassign == false) { out_string(c, "CLIENT_ERROR slab reassignment disabled"); return; } src = strtol(tokens[2].value, NULL, 10); dst = strtol(tokens[3].value, NULL, 10); if (errno == ERANGE) { out_string(c, "CLIENT_ERROR bad command line format"); return; } rv = slabs_reassign(src, dst); switch (rv) { case REASSIGN_OK: out_string(c, "OK"); break; case REASSIGN_RUNNING: out_string(c, "BUSY currently processing reassign request"); break; case REASSIGN_BADCLASS: out_string(c, "BADCLASS invalid src or dst class id"); break; case REASSIGN_NOSPARE: out_string(c, "NOSPARE source class has no spare pages"); break; case REASSIGN_SRC_DST_SAME: out_string(c, "SAME src and dst class are identical"); break; } return; } else if (ntokens >= 4 && (strcmp(tokens[COMMAND_TOKEN + 1].value, "automove") == 0)) { process_slabs_automove_command(c, tokens, ntokens); } else { out_string(c, "ERROR"); } } else if (ntokens > 1 && strcmp(tokens[COMMAND_TOKEN].value, "lru_crawler") == 0) { if (ntokens == 4 && strcmp(tokens[COMMAND_TOKEN + 1].value, "crawl") == 0) { int rv; if (settings.lru_crawler == false) { out_string(c, "CLIENT_ERROR lru crawler disabled"); return; } rv = lru_crawler_crawl(tokens[2].value, CRAWLER_EXPIRED, NULL, 0, settings.lru_crawler_tocrawl); switch(rv) { case CRAWLER_OK: out_string(c, "OK"); break; case CRAWLER_RUNNING: out_string(c, "BUSY currently processing crawler request"); break; case CRAWLER_BADCLASS: out_string(c, "BADCLASS invalid class id"); break; case CRAWLER_NOTSTARTED: out_string(c, "NOTSTARTED no items to crawl"); break; case CRAWLER_ERROR: out_string(c, "ERROR an unknown error happened"); break; } return; } else if (ntokens == 4 && strcmp(tokens[COMMAND_TOKEN + 1].value, "metadump") == 0) { if (settings.lru_crawler == false) { out_string(c, "CLIENT_ERROR lru crawler disabled"); return; } if (!settings.dump_enabled) { out_string(c, "ERROR metadump not allowed"); return; } int rv = lru_crawler_crawl(tokens[2].value, CRAWLER_METADUMP, c, c->sfd, LRU_CRAWLER_CAP_REMAINING); switch(rv) { case CRAWLER_OK: out_string(c, "OK"); // TODO: Don't reuse conn_watch here. conn_set_state(c, conn_watch); event_del(&c->event); break; case CRAWLER_RUNNING: out_string(c, "BUSY currently processing crawler request"); break; case CRAWLER_BADCLASS: out_string(c, "BADCLASS invalid class id"); break; case CRAWLER_NOTSTARTED: out_string(c, "NOTSTARTED no items to crawl"); break; case CRAWLER_ERROR: out_string(c, "ERROR an unknown error happened"); break; } return; } else if (ntokens == 4 && strcmp(tokens[COMMAND_TOKEN + 1].value, "tocrawl") == 0) { uint32_t tocrawl; if (!safe_strtoul(tokens[2].value, &tocrawl)) { out_string(c, "CLIENT_ERROR bad command line format"); return; } settings.lru_crawler_tocrawl = tocrawl; out_string(c, "OK"); return; } else if (ntokens == 4 && strcmp(tokens[COMMAND_TOKEN + 1].value, "sleep") == 0) { uint32_t tosleep; if (!safe_strtoul(tokens[2].value, &tosleep)) { out_string(c, "CLIENT_ERROR bad command line format"); return; } if (tosleep > 1000000) { out_string(c, "CLIENT_ERROR sleep must be one second or less"); return; } settings.lru_crawler_sleep = tosleep; out_string(c, "OK"); return; } else if (ntokens == 3) { if ((strcmp(tokens[COMMAND_TOKEN + 1].value, "enable") == 0)) { if (start_item_crawler_thread() == 0) { out_string(c, "OK"); } else { out_string(c, "ERROR failed to start lru crawler thread"); } } else if ((strcmp(tokens[COMMAND_TOKEN + 1].value, "disable") == 0)) { if (stop_item_crawler_thread(CRAWLER_NOWAIT) == 0) { out_string(c, "OK"); } else { out_string(c, "ERROR failed to stop lru crawler thread"); } } else { out_string(c, "ERROR"); } return; } else { out_string(c, "ERROR"); } } else if (ntokens > 1 && strcmp(tokens[COMMAND_TOKEN].value, "watch") == 0) { process_watch_command(c, tokens, ntokens); } else if ((ntokens == 3 || ntokens == 4) && (strcmp(tokens[COMMAND_TOKEN].value, "cache_memlimit") == 0)) { process_memlimit_command(c, tokens, ntokens); } else if ((ntokens == 3 || ntokens == 4) && (strcmp(tokens[COMMAND_TOKEN].value, "verbosity") == 0)) { process_verbosity_command(c, tokens, ntokens); } else if (ntokens >= 3 && strcmp(tokens[COMMAND_TOKEN].value, "lru") == 0) { process_lru_command(c, tokens, ntokens); #ifdef MEMCACHED_DEBUG // commands which exist only for testing the memcached's security protection } else if (ntokens == 2 && (strcmp(tokens[COMMAND_TOKEN].value, "misbehave") == 0)) { process_misbehave_command(c); #endif #ifdef EXTSTORE } else if (ntokens >= 3 && strcmp(tokens[COMMAND_TOKEN].value, "extstore") == 0) { process_extstore_command(c, tokens, ntokens); #endif #ifdef TLS } else if (ntokens == 2 && strcmp(tokens[COMMAND_TOKEN].value, "refresh_certs") == 0) { set_noreply_maybe(c, tokens, ntokens); char *errmsg = NULL; if (refresh_certs(&errmsg)) { out_string(c, "OK"); } else { write_and_free(c, errmsg, strlen(errmsg)); } return; #endif } else { if (ntokens >= 2 && strncmp(tokens[ntokens - 2].value, "HTTP/", 5) == 0) { conn_set_state(c, conn_closing); } else { out_string(c, "ERROR"); } } return; } static int try_read_command_negotiate(conn *c) { assert(c->protocol == negotiating_prot); assert(c != NULL); assert(c->rcurr <= (c->rbuf + c->rsize)); assert(c->rbytes > 0); if ((unsigned char)c->rbuf[0] == (unsigned char)PROTOCOL_BINARY_REQ) { c->protocol = binary_prot; c->try_read_command = try_read_command_binary; } else { // authentication doesn't work with negotiated protocol. c->protocol = ascii_prot; c->try_read_command = try_read_command_ascii; } if (settings.verbose > 1) { fprintf(stderr, "%d: Client using the %s protocol\n", c->sfd, prot_text(c->protocol)); } return c->try_read_command(c); } static int try_read_command_udp(conn *c) { assert(c != NULL); assert(c->rcurr <= (c->rbuf + c->rsize)); assert(c->rbytes > 0); if ((unsigned char)c->rbuf[0] == (unsigned char)PROTOCOL_BINARY_REQ) { c->protocol = binary_prot; return try_read_command_binary(c); } else { c->protocol = ascii_prot; return try_read_command_ascii(c); } } static int try_read_command_binary(conn *c) { /* Do we have the complete packet header? */ if (c->rbytes < sizeof(c->binary_header)) { /* need more data! */ return 0; } else { #ifdef NEED_ALIGN if (((long)(c->rcurr)) % 8 != 0) { /* must realign input buffer */ memmove(c->rbuf, c->rcurr, c->rbytes); c->rcurr = c->rbuf; if (settings.verbose > 1) { fprintf(stderr, "%d: Realign input buffer\n", c->sfd); } } #endif protocol_binary_request_header* req; req = (protocol_binary_request_header*)c->rcurr; if (settings.verbose > 1) { /* Dump the packet before we convert it to host order */ int ii; fprintf(stderr, "<%d Read binary protocol data:", c->sfd); for (ii = 0; ii < sizeof(req->bytes); ++ii) { if (ii % 4 == 0) { fprintf(stderr, "\n<%d ", c->sfd); } fprintf(stderr, " 0x%02x", req->bytes[ii]); } fprintf(stderr, "\n"); } c->binary_header = *req; c->binary_header.request.keylen = ntohs(req->request.keylen); c->binary_header.request.bodylen = ntohl(req->request.bodylen); c->binary_header.request.cas = ntohll(req->request.cas); if (c->binary_header.request.magic != PROTOCOL_BINARY_REQ) { if (settings.verbose) { fprintf(stderr, "Invalid magic: %x\n", c->binary_header.request.magic); } conn_set_state(c, conn_closing); return -1; } c->msgcurr = 0; c->msgused = 0; c->iovused = 0; if (add_msghdr(c) != 0) { out_of_memory(c, "SERVER_ERROR Out of memory allocating headers"); return 0; } c->cmd = c->binary_header.request.opcode; c->keylen = c->binary_header.request.keylen; c->opaque = c->binary_header.request.opaque; /* clear the returned cas value */ c->cas = 0; c->last_cmd_time = current_time; dispatch_bin_command(c); c->rbytes -= sizeof(c->binary_header); c->rcurr += sizeof(c->binary_header); } return 1; } static int try_read_command_asciiauth(conn *c) { token_t tokens[MAX_TOKENS]; size_t ntokens; char *cont = NULL; // TODO: move to another function. if (!c->sasl_started) { char *el; uint32_t size = 0; // impossible for the auth command to be this short. if (c->rbytes < 2) return 0; el = memchr(c->rcurr, '\n', c->rbytes); // If no newline after 1k, getting junk data, close out. if (!el) { if (c->rbytes > 1024) { conn_set_state(c, conn_closing); return 1; } return 0; } // Looking for: "set foo 0 0 N\r\nuser pass\r\n" // key, flags, and ttl are ignored. N is used to see if we have the rest. // so tokenize doesn't walk past into the value. // it's fine to leave the \r in, as strtoul will stop at it. *el = '\0'; ntokens = tokenize_command(c->rcurr, tokens, MAX_TOKENS); // ensure the buffer is consumed. c->rbytes -= (el - c->rcurr) + 1; c->rcurr += (el - c->rcurr) + 1; // final token is a NULL ender, so we have one more than expected. if (ntokens < 6 || strcmp(tokens[0].value, "set") != 0 || !safe_strtoul(tokens[4].value, &size)) { out_string(c, "CLIENT_ERROR unauthenticated"); return 1; } // we don't actually care about the key at all; it can be anything. // we do care about the size of the remaining read. c->rlbytes = size + 2; c->sasl_started = true; // reuse from binprot sasl, but not sasl :) } if (c->rbytes < c->rlbytes) { // need more bytes. return 0; } cont = c->rcurr; // advance buffer. no matter what we're stopping. c->rbytes -= c->rlbytes; c->rcurr += c->rlbytes; c->sasl_started = false; // must end with \r\n // NB: I thought ASCII sets also worked with just \n, but according to // complete_nread_ascii only \r\n is valid. if (strncmp(cont + c->rlbytes - 2, "\r\n", 2) != 0) { out_string(c, "CLIENT_ERROR bad command line termination"); return 1; } // payload should be "user pass", so we can use the tokenizer. cont[c->rlbytes - 2] = '\0'; ntokens = tokenize_command(cont, tokens, MAX_TOKENS); if (ntokens < 3) { out_string(c, "CLIENT_ERROR bad authentication token format"); return 1; } if (authfile_check(tokens[0].value, tokens[1].value) == 1) { out_string(c, "STORED"); c->authenticated = true; c->try_read_command = try_read_command_ascii; pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.auth_cmds++; pthread_mutex_unlock(&c->thread->stats.mutex); } else { out_string(c, "CLIENT_ERROR authentication failure"); pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.auth_cmds++; c->thread->stats.auth_errors++; pthread_mutex_unlock(&c->thread->stats.mutex); } return 1; } static int try_read_command_ascii(conn *c) { char *el, *cont; if (c->rbytes == 0) return 0; el = memchr(c->rcurr, '\n', c->rbytes); if (!el) { if (c->rbytes > 1024) { /* * We didn't have a '\n' in the first k. This _has_ to be a * large multiget, if not we should just nuke the connection. */ char *ptr = c->rcurr; while (*ptr == ' ') { /* ignore leading whitespaces */ ++ptr; } if (ptr - c->rcurr > 100 || (strncmp(ptr, "get ", 4) && strncmp(ptr, "gets ", 5))) { conn_set_state(c, conn_closing); return 1; } } return 0; } cont = el + 1; if ((el - c->rcurr) > 1 && *(el - 1) == '\r') { el--; } *el = '\0'; assert(cont <= (c->rcurr + c->rbytes)); c->last_cmd_time = current_time; process_command(c, c->rcurr); c->rbytes -= (cont - c->rcurr); c->rcurr = cont; assert(c->rcurr <= (c->rbuf + c->rsize)); return 1; } /* * read a UDP request. */ static enum try_read_result try_read_udp(conn *c) { int res; assert(c != NULL); c->request_addr_size = sizeof(c->request_addr); res = recvfrom(c->sfd, c->rbuf, c->rsize, 0, (struct sockaddr *)&c->request_addr, &c->request_addr_size); if (res > 8) { unsigned char *buf = (unsigned char *)c->rbuf; pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.bytes_read += res; pthread_mutex_unlock(&c->thread->stats.mutex); /* Beginning of UDP packet is the request ID; save it. */ c->request_id = buf[0] * 256 + buf[1]; /* If this is a multi-packet request, drop it. */ if (buf[4] != 0 || buf[5] != 1) { out_string(c, "SERVER_ERROR multi-packet request not supported"); return READ_NO_DATA_RECEIVED; } /* Don't care about any of the rest of the header. */ res -= 8; memmove(c->rbuf, c->rbuf + 8, res); c->rbytes = res; c->rcurr = c->rbuf; return READ_DATA_RECEIVED; } return READ_NO_DATA_RECEIVED; } /* * read from network as much as we can, handle buffer overflow and connection * close. * before reading, move the remaining incomplete fragment of a command * (if any) to the beginning of the buffer. * * To protect us from someone flooding a connection with bogus data causing * the connection to eat up all available memory, break out and start looking * at the data I've got after a number of reallocs... * * @return enum try_read_result */ static enum try_read_result try_read_network(conn *c) { enum try_read_result gotdata = READ_NO_DATA_RECEIVED; int res; int num_allocs = 0; assert(c != NULL); if (c->rcurr != c->rbuf) { if (c->rbytes != 0) /* otherwise there's nothing to copy */ memmove(c->rbuf, c->rcurr, c->rbytes); c->rcurr = c->rbuf; } while (1) { if (c->rbytes >= c->rsize) { if (num_allocs == 4) { return gotdata; } ++num_allocs; char *new_rbuf = realloc(c->rbuf, c->rsize * 2); if (!new_rbuf) { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); if (settings.verbose > 0) { fprintf(stderr, "Couldn't realloc input buffer\n"); } c->rbytes = 0; /* ignore what we read */ out_of_memory(c, "SERVER_ERROR out of memory reading request"); c->write_and_go = conn_closing; return READ_MEMORY_ERROR; } c->rcurr = c->rbuf = new_rbuf; c->rsize *= 2; } int avail = c->rsize - c->rbytes; res = c->read(c, c->rbuf + c->rbytes, avail); if (res > 0) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.bytes_read += res; pthread_mutex_unlock(&c->thread->stats.mutex); gotdata = READ_DATA_RECEIVED; c->rbytes += res; if (res == avail) { continue; } else { break; } } if (res == 0) { return READ_ERROR; } if (res == -1) { if (errno == EAGAIN || errno == EWOULDBLOCK) { break; } return READ_ERROR; } } return gotdata; } static bool update_event(conn *c, const int new_flags) { assert(c != NULL); struct event_base *base = c->event.ev_base; if (c->ev_flags == new_flags) return true; if (event_del(&c->event) == -1) return false; event_set(&c->event, c->sfd, new_flags, event_handler, (void *)c); event_base_set(base, &c->event); c->ev_flags = new_flags; if (event_add(&c->event, 0) == -1) return false; return true; } /* * Sets whether we are listening for new connections or not. */ void do_accept_new_conns(const bool do_accept) { conn *next; for (next = listen_conn; next; next = next->next) { if (do_accept) { update_event(next, EV_READ | EV_PERSIST); if (listen(next->sfd, settings.backlog) != 0) { perror("listen"); } } else { update_event(next, 0); if (listen(next->sfd, 0) != 0) { perror("listen"); } } } if (do_accept) { struct timeval maxconns_exited; uint64_t elapsed_us; gettimeofday(&maxconns_exited,NULL); STATS_LOCK(); elapsed_us = (maxconns_exited.tv_sec - stats.maxconns_entered.tv_sec) * 1000000 + (maxconns_exited.tv_usec - stats.maxconns_entered.tv_usec); stats.time_in_listen_disabled_us += elapsed_us; stats_state.accepting_conns = true; STATS_UNLOCK(); } else { STATS_LOCK(); stats_state.accepting_conns = false; gettimeofday(&stats.maxconns_entered,NULL); stats.listen_disabled_num++; STATS_UNLOCK(); allow_new_conns = false; maxconns_handler(-42, 0, 0); } } /* * Transmit the next chunk of data from our list of msgbuf structures. * * Returns: * TRANSMIT_COMPLETE All done writing. * TRANSMIT_INCOMPLETE More data remaining to write. * TRANSMIT_SOFT_ERROR Can't write any more right now. * TRANSMIT_HARD_ERROR Can't write (c->state is set to conn_closing) */ static enum transmit_result transmit(conn *c) { assert(c != NULL); if (c->msgcurr < c->msgused && c->msglist[c->msgcurr].msg_iovlen == 0) { /* Finished writing the current msg; advance to the next. */ c->msgcurr++; } if (c->msgcurr < c->msgused) { ssize_t res; struct msghdr *m = &c->msglist[c->msgcurr]; res = c->sendmsg(c, m, 0); if (res >= 0) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.bytes_written += res; pthread_mutex_unlock(&c->thread->stats.mutex); /* We've written some of the data. Remove the completed iovec entries from the list of pending writes. */ while (m->msg_iovlen > 0 && res >= m->msg_iov->iov_len) { res -= m->msg_iov->iov_len; m->msg_iovlen--; m->msg_iov++; } /* Might have written just part of the last iovec entry; adjust it so the next write will do the rest. */ if (res > 0) { m->msg_iov->iov_base = (caddr_t)m->msg_iov->iov_base + res; m->msg_iov->iov_len -= res; } return TRANSMIT_INCOMPLETE; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_WRITE | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); conn_set_state(c, conn_closing); return TRANSMIT_HARD_ERROR; } return TRANSMIT_SOFT_ERROR; } /* if res == -1 and error is not EAGAIN or EWOULDBLOCK, we have a real error, on which we close the connection */ if (settings.verbose > 0) perror("Failed to write, and not due to blocking"); if (IS_UDP(c->transport)) conn_set_state(c, conn_read); else conn_set_state(c, conn_closing); return TRANSMIT_HARD_ERROR; } else { return TRANSMIT_COMPLETE; } } /* Does a looped read to fill data chunks */ /* TODO: restrict number of times this can loop. * Also, benchmark using readv's. */ static int read_into_chunked_item(conn *c) { int total = 0; int res; assert(c->rcurr != c->ritem); while (c->rlbytes > 0) { item_chunk *ch = (item_chunk *)c->ritem; if (ch->size == ch->used) { // FIXME: ch->next is currently always 0. remove this? if (ch->next) { c->ritem = (char *) ch->next; } else { /* Allocate next chunk. Binary protocol needs 2b for \r\n */ c->ritem = (char *) do_item_alloc_chunk(ch, c->rlbytes + ((c->protocol == binary_prot) ? 2 : 0)); if (!c->ritem) { // We failed an allocation. Let caller handle cleanup. total = -2; break; } // ritem has new chunk, restart the loop. continue; //assert(c->rlbytes == 0); } } int unused = ch->size - ch->used; /* first check if we have leftovers in the conn_read buffer */ if (c->rbytes > 0) { total = 0; int tocopy = c->rbytes > c->rlbytes ? c->rlbytes : c->rbytes; tocopy = tocopy > unused ? unused : tocopy; if (c->ritem != c->rcurr) { memmove(ch->data + ch->used, c->rcurr, tocopy); } total += tocopy; c->rlbytes -= tocopy; c->rcurr += tocopy; c->rbytes -= tocopy; ch->used += tocopy; if (c->rlbytes == 0) { break; } } else { /* now try reading from the socket */ res = c->read(c, ch->data + ch->used, (unused > c->rlbytes ? c->rlbytes : unused)); if (res > 0) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.bytes_read += res; pthread_mutex_unlock(&c->thread->stats.mutex); ch->used += res; total += res; c->rlbytes -= res; } else { /* Reset total to the latest result so caller can handle it */ total = res; break; } } } /* At some point I will be able to ditch the \r\n from item storage and remove all of these kludges. The above binprot check ensures inline space for \r\n, but if we do exactly enough allocs there will be no additional chunk for \r\n. */ if (c->rlbytes == 0 && c->protocol == binary_prot && total >= 0) { item_chunk *ch = (item_chunk *)c->ritem; if (ch->size - ch->used < 2) { c->ritem = (char *) do_item_alloc_chunk(ch, 2); if (!c->ritem) { total = -2; } } } return total; } static void drive_machine(conn *c) { bool stop = false; int sfd; socklen_t addrlen; struct sockaddr_storage addr; int nreqs = settings.reqs_per_event; int res; const char *str; #ifdef HAVE_ACCEPT4 static int use_accept4 = 1; #else static int use_accept4 = 0; #endif assert(c != NULL); while (!stop) { switch(c->state) { case conn_listening: addrlen = sizeof(addr); #ifdef HAVE_ACCEPT4 if (use_accept4) { sfd = accept4(c->sfd, (struct sockaddr *)&addr, &addrlen, SOCK_NONBLOCK); } else { sfd = accept(c->sfd, (struct sockaddr *)&addr, &addrlen); } #else sfd = accept(c->sfd, (struct sockaddr *)&addr, &addrlen); #endif if (sfd == -1) { if (use_accept4 && errno == ENOSYS) { use_accept4 = 0; continue; } perror(use_accept4 ? "accept4()" : "accept()"); if (errno == EAGAIN || errno == EWOULDBLOCK) { /* these are transient, so don't log anything */ stop = true; } else if (errno == EMFILE) { if (settings.verbose > 0) fprintf(stderr, "Too many open connections\n"); accept_new_conns(false); stop = true; } else { perror("accept()"); stop = true; } break; } if (!use_accept4) { if (fcntl(sfd, F_SETFL, fcntl(sfd, F_GETFL) | O_NONBLOCK) < 0) { perror("setting O_NONBLOCK"); close(sfd); break; } } bool reject; if (settings.maxconns_fast) { STATS_LOCK(); reject = stats_state.curr_conns + stats_state.reserved_fds >= settings.maxconns - 1; stats.rejected_conns++; STATS_UNLOCK(); } else { reject = false; } if (reject) { str = "ERROR Too many open connections\r\n"; res = write(sfd, str, strlen(str)); close(sfd); } else { void *ssl_v = NULL; #ifdef TLS SSL *ssl = NULL; if (c->ssl_enabled) { assert(IS_TCP(c->transport) && settings.ssl_enabled); if (settings.ssl_ctx == NULL) { if (settings.verbose) { fprintf(stderr, "SSL context is not initialized\n"); } close(sfd); break; } SSL_LOCK(); ssl = SSL_new(settings.ssl_ctx); SSL_UNLOCK(); if (ssl == NULL) { if (settings.verbose) { fprintf(stderr, "Failed to created the SSL object\n"); } close(sfd); break; } SSL_set_fd(ssl, sfd); int ret = SSL_accept(ssl); if (ret < 0) { int err = SSL_get_error(ssl, ret); if (err == SSL_ERROR_SYSCALL || err == SSL_ERROR_SSL) { if (settings.verbose) { fprintf(stderr, "SSL connection failed with error code : %d : %s\n", err, strerror(errno)); } SSL_free(ssl); close(sfd); break; } } } ssl_v = (void*) ssl; #endif dispatch_conn_new(sfd, conn_new_cmd, EV_READ | EV_PERSIST, DATA_BUFFER_SIZE, c->transport, ssl_v); } stop = true; break; case conn_waiting: if (!update_event(c, EV_READ | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); conn_set_state(c, conn_closing); break; } conn_set_state(c, conn_read); stop = true; break; case conn_read: res = IS_UDP(c->transport) ? try_read_udp(c) : try_read_network(c); switch (res) { case READ_NO_DATA_RECEIVED: conn_set_state(c, conn_waiting); break; case READ_DATA_RECEIVED: conn_set_state(c, conn_parse_cmd); break; case READ_ERROR: conn_set_state(c, conn_closing); break; case READ_MEMORY_ERROR: /* Failed to allocate more memory */ /* State already set by try_read_network */ break; } break; case conn_parse_cmd : if (c->try_read_command(c) == 0) { /* wee need more data! */ conn_set_state(c, conn_waiting); } break; case conn_new_cmd: /* Only process nreqs at a time to avoid starving other connections */ --nreqs; if (nreqs >= 0) { reset_cmd_handler(c); } else { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.conn_yields++; pthread_mutex_unlock(&c->thread->stats.mutex); if (c->rbytes > 0) { /* We have already read in data into the input buffer, so libevent will most likely not signal read events on the socket (unless more data is available. As a hack we should just put in a request to write data, because that should be possible ;-) */ if (!update_event(c, EV_WRITE | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); conn_set_state(c, conn_closing); break; } } stop = true; } break; case conn_nread: if (c->rlbytes == 0) { complete_nread(c); break; } /* Check if rbytes < 0, to prevent crash */ if (c->rlbytes < 0) { if (settings.verbose) { fprintf(stderr, "Invalid rlbytes to read: len %d\n", c->rlbytes); } conn_set_state(c, conn_closing); break; } if (!c->item || (((item *)c->item)->it_flags & ITEM_CHUNKED) == 0) { /* first check if we have leftovers in the conn_read buffer */ if (c->rbytes > 0) { int tocopy = c->rbytes > c->rlbytes ? c->rlbytes : c->rbytes; if (c->ritem != c->rcurr) { memmove(c->ritem, c->rcurr, tocopy); } c->ritem += tocopy; c->rlbytes -= tocopy; c->rcurr += tocopy; c->rbytes -= tocopy; if (c->rlbytes == 0) { break; } } /* now try reading from the socket */ res = c->read(c, c->ritem, c->rlbytes); if (res > 0) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.bytes_read += res; pthread_mutex_unlock(&c->thread->stats.mutex); if (c->rcurr == c->ritem) { c->rcurr += res; } c->ritem += res; c->rlbytes -= res; break; } } else { res = read_into_chunked_item(c); if (res > 0) break; } if (res == 0) { /* end of stream */ conn_set_state(c, conn_closing); break; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_READ | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); conn_set_state(c, conn_closing); break; } stop = true; break; } /* Memory allocation failure */ if (res == -2) { out_of_memory(c, "SERVER_ERROR Out of memory during read"); c->sbytes = c->rlbytes; c->write_and_go = conn_swallow; // Ensure this flag gets cleared. It gets killed on conn_new() // so any conn_closing is fine, calling complete_nread is // fine. This swallow semms to be the only other case. c->set_stale = false; c->mset_res = false; break; } /* otherwise we have a real error, on which we close the connection */ if (settings.verbose > 0) { fprintf(stderr, "Failed to read, and not due to blocking:\n" "errno: %d %s \n" "rcurr=%lx ritem=%lx rbuf=%lx rlbytes=%d rsize=%d\n", errno, strerror(errno), (long)c->rcurr, (long)c->ritem, (long)c->rbuf, (int)c->rlbytes, (int)c->rsize); } conn_set_state(c, conn_closing); break; case conn_swallow: /* we are reading sbytes and throwing them away */ if (c->sbytes <= 0) { conn_set_state(c, conn_new_cmd); break; } /* first check if we have leftovers in the conn_read buffer */ if (c->rbytes > 0) { int tocopy = c->rbytes > c->sbytes ? c->sbytes : c->rbytes; c->sbytes -= tocopy; c->rcurr += tocopy; c->rbytes -= tocopy; break; } /* now try reading from the socket */ res = c->read(c, c->rbuf, c->rsize > c->sbytes ? c->sbytes : c->rsize); if (res > 0) { pthread_mutex_lock(&c->thread->stats.mutex); c->thread->stats.bytes_read += res; pthread_mutex_unlock(&c->thread->stats.mutex); c->sbytes -= res; break; } if (res == 0) { /* end of stream */ conn_set_state(c, conn_closing); break; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_READ | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); conn_set_state(c, conn_closing); break; } stop = true; break; } /* otherwise we have a real error, on which we close the connection */ if (settings.verbose > 0) fprintf(stderr, "Failed to read, and not due to blocking\n"); conn_set_state(c, conn_closing); break; case conn_write: /* * We want to write out a simple response. If we haven't already, * assemble it into a msgbuf list (this will be a single-entry * list for TCP or a two-entry list for UDP). */ if (c->iovused == 0 || (IS_UDP(c->transport) && c->iovused == 1)) { if (add_iov(c, c->wcurr, c->wbytes) != 0) { if (settings.verbose > 0) fprintf(stderr, "Couldn't build response\n"); conn_set_state(c, conn_closing); break; } } /* fall through... */ case conn_mwrite: #ifdef EXTSTORE /* have side IO's that must process before transmit() can run. * remove the connection from the worker thread and dispatch the * IO queue */ if (c->io_wrapleft) { assert(c->io_queued == false); assert(c->io_wraplist != NULL); // TODO: create proper state for this condition conn_set_state(c, conn_watch); event_del(&c->event); c->io_queued = true; extstore_submit(c->thread->storage, &c->io_wraplist->io); stop = true; break; } #endif if (IS_UDP(c->transport) && c->msgcurr == 0 && build_udp_headers(c) != 0) { if (settings.verbose > 0) fprintf(stderr, "Failed to build UDP headers\n"); conn_set_state(c, conn_closing); break; } switch (transmit(c)) { case TRANSMIT_COMPLETE: if (c->state == conn_mwrite) { conn_release_items(c); /* XXX: I don't know why this wasn't the general case */ if(c->protocol == binary_prot) { conn_set_state(c, c->write_and_go); } else { conn_set_state(c, conn_new_cmd); } } else if (c->state == conn_write) { if (c->write_and_free) { free(c->write_and_free); c->write_and_free = 0; } conn_set_state(c, c->write_and_go); } else { if (settings.verbose > 0) fprintf(stderr, "Unexpected state %d\n", c->state); conn_set_state(c, conn_closing); } break; case TRANSMIT_INCOMPLETE: case TRANSMIT_HARD_ERROR: break; /* Continue in state machine. */ case TRANSMIT_SOFT_ERROR: stop = true; break; } break; case conn_closing: if (IS_UDP(c->transport)) conn_cleanup(c); else conn_close(c); stop = true; break; case conn_closed: /* This only happens if dormando is an idiot. */ abort(); break; case conn_watch: /* We handed off our connection to the logger thread. */ stop = true; break; case conn_max_state: assert(false); break; } } return; } void event_handler(const int fd, const short which, void *arg) { conn *c; c = (conn *)arg; assert(c != NULL); c->which = which; /* sanity */ if (fd != c->sfd) { if (settings.verbose > 0) fprintf(stderr, "Catastrophic: event fd doesn't match conn fd!\n"); conn_close(c); return; } drive_machine(c); /* wait for next event */ return; } static int new_socket(struct addrinfo *ai) { int sfd; int flags; if ((sfd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol)) == -1) { return -1; } if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 || fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) { perror("setting O_NONBLOCK"); close(sfd); return -1; } return sfd; } /* * Sets a socket's send buffer size to the maximum allowed by the system. */ static void maximize_sndbuf(const int sfd) { socklen_t intsize = sizeof(int); int last_good = 0; int min, max, avg; int old_size; /* Start with the default size. */ if (getsockopt(sfd, SOL_SOCKET, SO_SNDBUF, &old_size, &intsize) != 0) { if (settings.verbose > 0) perror("getsockopt(SO_SNDBUF)"); return; } /* Binary-search for the real maximum. */ min = old_size; max = MAX_SENDBUF_SIZE; while (min <= max) { avg = ((unsigned int)(min + max)) / 2; if (setsockopt(sfd, SOL_SOCKET, SO_SNDBUF, (void *)&avg, intsize) == 0) { last_good = avg; min = avg + 1; } else { max = avg - 1; } } if (settings.verbose > 1) fprintf(stderr, "<%d send buffer was %d, now %d\n", sfd, old_size, last_good); } /** * Create a socket and bind it to a specific port number * @param interface the interface to bind to * @param port the port number to bind to * @param transport the transport protocol (TCP / UDP) * @param portnumber_file A filepointer to write the port numbers to * when they are successfully added to the list of ports we * listen on. */ static int server_socket(const char *interface, int port, enum network_transport transport, FILE *portnumber_file, bool ssl_enabled) { int sfd; struct linger ling = {0, 0}; struct addrinfo *ai; struct addrinfo *next; struct addrinfo hints = { .ai_flags = AI_PASSIVE, .ai_family = AF_UNSPEC }; char port_buf[NI_MAXSERV]; int error; int success = 0; int flags =1; hints.ai_socktype = IS_UDP(transport) ? SOCK_DGRAM : SOCK_STREAM; if (port == -1) { port = 0; } snprintf(port_buf, sizeof(port_buf), "%d", port); error= getaddrinfo(interface, port_buf, &hints, &ai); if (error != 0) { if (error != EAI_SYSTEM) fprintf(stderr, "getaddrinfo(): %s\n", gai_strerror(error)); else perror("getaddrinfo()"); return 1; } for (next= ai; next; next= next->ai_next) { conn *listen_conn_add; if ((sfd = new_socket(next)) == -1) { /* getaddrinfo can return "junk" addresses, * we make sure at least one works before erroring. */ if (errno == EMFILE) { /* ...unless we're out of fds */ perror("server_socket"); exit(EX_OSERR); } continue; } #ifdef IPV6_V6ONLY if (next->ai_family == AF_INET6) { error = setsockopt(sfd, IPPROTO_IPV6, IPV6_V6ONLY, (char *) &flags, sizeof(flags)); if (error != 0) { perror("setsockopt"); close(sfd); continue; } } #endif setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flags, sizeof(flags)); if (IS_UDP(transport)) { maximize_sndbuf(sfd); } else { error = setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags)); if (error != 0) perror("setsockopt"); error = setsockopt(sfd, SOL_SOCKET, SO_LINGER, (void *)&ling, sizeof(ling)); if (error != 0) perror("setsockopt"); error = setsockopt(sfd, IPPROTO_TCP, TCP_NODELAY, (void *)&flags, sizeof(flags)); if (error != 0) perror("setsockopt"); } if (bind(sfd, next->ai_addr, next->ai_addrlen) == -1) { if (errno != EADDRINUSE) { perror("bind()"); close(sfd); freeaddrinfo(ai); return 1; } close(sfd); continue; } else { success++; if (!IS_UDP(transport) && listen(sfd, settings.backlog) == -1) { perror("listen()"); close(sfd); freeaddrinfo(ai); return 1; } if (portnumber_file != NULL && (next->ai_addr->sa_family == AF_INET || next->ai_addr->sa_family == AF_INET6)) { union { struct sockaddr_in in; struct sockaddr_in6 in6; } my_sockaddr; socklen_t len = sizeof(my_sockaddr); if (getsockname(sfd, (struct sockaddr*)&my_sockaddr, &len)==0) { if (next->ai_addr->sa_family == AF_INET) { fprintf(portnumber_file, "%s INET: %u\n", IS_UDP(transport) ? "UDP" : "TCP", ntohs(my_sockaddr.in.sin_port)); } else { fprintf(portnumber_file, "%s INET6: %u\n", IS_UDP(transport) ? "UDP" : "TCP", ntohs(my_sockaddr.in6.sin6_port)); } } } } if (IS_UDP(transport)) { int c; for (c = 0; c < settings.num_threads_per_udp; c++) { /* Allocate one UDP file descriptor per worker thread; * this allows "stats conns" to separately list multiple * parallel UDP requests in progress. * * The dispatch code round-robins new connection requests * among threads, so this is guaranteed to assign one * FD to each thread. */ int per_thread_fd; if (c == 0) { per_thread_fd = sfd; } else { per_thread_fd = dup(sfd); if (per_thread_fd < 0) { perror("Failed to duplicate file descriptor"); exit(EXIT_FAILURE); } } dispatch_conn_new(per_thread_fd, conn_read, EV_READ | EV_PERSIST, UDP_READ_BUFFER_SIZE, transport, NULL); } } else { if (!(listen_conn_add = conn_new(sfd, conn_listening, EV_READ | EV_PERSIST, 1, transport, main_base, NULL))) { fprintf(stderr, "failed to create listening connection\n"); exit(EXIT_FAILURE); } #ifdef TLS listen_conn_add->ssl_enabled = ssl_enabled; #else assert(ssl_enabled == false); #endif listen_conn_add->next = listen_conn; listen_conn = listen_conn_add; } } freeaddrinfo(ai); /* Return zero iff we detected no errors in starting up connections */ return success == 0; } static int server_sockets(int port, enum network_transport transport, FILE *portnumber_file) { bool ssl_enabled = false; #ifdef TLS const char *notls = "notls"; ssl_enabled = settings.ssl_enabled; #endif if (settings.inter == NULL) { return server_socket(settings.inter, port, transport, portnumber_file, ssl_enabled); } else { // tokenize them and bind to each one of them.. char *b; int ret = 0; char *list = strdup(settings.inter); if (list == NULL) { fprintf(stderr, "Failed to allocate memory for parsing server interface string\n"); return 1; } for (char *p = strtok_r(list, ";,", &b); p != NULL; p = strtok_r(NULL, ";,", &b)) { int the_port = port; #ifdef TLS ssl_enabled = settings.ssl_enabled; // "notls" option is valid only when memcached is run with SSL enabled. if (strncmp(p, notls, strlen(notls)) == 0) { if (!settings.ssl_enabled) { fprintf(stderr, "'notls' option is valid only when SSL is enabled\n"); free(list); return 1; } ssl_enabled = false; p += strlen(notls) + 1; } #endif char *h = NULL; if (*p == '[') { // expecting it to be an IPv6 address enclosed in [] // i.e. RFC3986 style recommended by RFC5952 char *e = strchr(p, ']'); if (e == NULL) { fprintf(stderr, "Invalid IPV6 address: \"%s\"", p); free(list); return 1; } h = ++p; // skip the opening '[' *e = '\0'; p = ++e; // skip the closing ']' } char *s = strchr(p, ':'); if (s != NULL) { // If no more semicolons - attempt to treat as port number. // Otherwise the only valid option is an unenclosed IPv6 without port, until // of course there was an RFC3986 IPv6 address previously specified - // in such a case there is no good option, will just send it to fail as port number. if (strchr(s + 1, ':') == NULL || h != NULL) { *s = '\0'; ++s; if (!safe_strtol(s, &the_port)) { fprintf(stderr, "Invalid port number: \"%s\"", s); free(list); return 1; } } } if (h != NULL) p = h; if (strcmp(p, "*") == 0) { p = NULL; } ret |= server_socket(p, the_port, transport, portnumber_file, ssl_enabled); } free(list); return ret; } } static int new_socket_unix(void) { int sfd; int flags; if ((sfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { perror("socket()"); return -1; } if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 || fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) { perror("setting O_NONBLOCK"); close(sfd); return -1; } return sfd; } static int server_socket_unix(const char *path, int access_mask) { int sfd; struct linger ling = {0, 0}; struct sockaddr_un addr; struct stat tstat; int flags =1; int old_umask; if (!path) { return 1; } if ((sfd = new_socket_unix()) == -1) { return 1; } /* * Clean up a previous socket file if we left it around */ if (lstat(path, &tstat) == 0) { if (S_ISSOCK(tstat.st_mode)) unlink(path); } setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flags, sizeof(flags)); setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags)); setsockopt(sfd, SOL_SOCKET, SO_LINGER, (void *)&ling, sizeof(ling)); /* * the memset call clears nonstandard fields in some implementations * that otherwise mess things up. */ memset(&addr, 0, sizeof(addr)); addr.sun_family = AF_UNIX; strncpy(addr.sun_path, path, sizeof(addr.sun_path) - 1); assert(strcmp(addr.sun_path, path) == 0); old_umask = umask( ~(access_mask&0777)); if (bind(sfd, (struct sockaddr *)&addr, sizeof(addr)) == -1) { perror("bind()"); close(sfd); umask(old_umask); return 1; } umask(old_umask); if (listen(sfd, settings.backlog) == -1) { perror("listen()"); close(sfd); return 1; } if (!(listen_conn = conn_new(sfd, conn_listening, EV_READ | EV_PERSIST, 1, local_transport, main_base, NULL))) { fprintf(stderr, "failed to create listening connection\n"); exit(EXIT_FAILURE); } return 0; } /* * We keep the current time of day in a global variable that's updated by a * timer event. This saves us a bunch of time() system calls (we really only * need to get the time once a second, whereas there can be tens of thousands * of requests a second) and allows us to use server-start-relative timestamps * rather than absolute UNIX timestamps, a space savings on systems where * sizeof(time_t) > sizeof(unsigned int). */ volatile rel_time_t current_time; static struct event clockevent; #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) static bool monotonic = false; static int64_t monotonic_start; #endif /* libevent uses a monotonic clock when available for event scheduling. Aside * from jitter, simply ticking our internal timer here is accurate enough. * Note that users who are setting explicit dates for expiration times *must* * ensure their clocks are correct before starting memcached. */ static void clock_handler(const int fd, const short which, void *arg) { struct timeval t = {.tv_sec = 1, .tv_usec = 0}; static bool initialized = false; if (initialized) { /* only delete the event if it's actually there. */ evtimer_del(&clockevent); } else { initialized = true; } // While we're here, check for hash table expansion. // This function should be quick to avoid delaying the timer. assoc_start_expand(stats_state.curr_items); // also, if HUP'ed we need to do some maintenance. // for now that's just the authfile reload. if (settings.sig_hup) { settings.sig_hup = false; authfile_load(settings.auth_file); } evtimer_set(&clockevent, clock_handler, 0); event_base_set(main_base, &clockevent); evtimer_add(&clockevent, &t); #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) if (monotonic) { struct timespec ts; if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) return; current_time = (rel_time_t) (ts.tv_sec - monotonic_start); return; } #endif { struct timeval tv; gettimeofday(&tv, NULL); current_time = (rel_time_t) (tv.tv_sec - process_started); } } static const char* flag_enabled_disabled(bool flag) { return (flag ? "enabled" : "disabled"); } static void verify_default(const char* param, bool condition) { if (!condition) { printf("Default value of [%s] has changed." " Modify the help text and default value check.\n", param); exit(EXIT_FAILURE); } } static void usage(void) { printf(PACKAGE " " VERSION "\n"); printf("-p, --port= TCP port to listen on (default: %d)\n" "-U, --udp-port= UDP port to listen on (default: %d, off)\n" "-s, --unix-socket= UNIX socket to listen on (disables network support)\n" "-A, --enable-shutdown enable ascii \"shutdown\" command\n" "-a, --unix-mask= access mask for UNIX socket, in octal (default: %o)\n" "-l, --listen= interface to listen on (default: INADDR_ANY)\n" #ifdef TLS " if TLS/SSL is enabled, 'notls' prefix can be used to\n" " disable for specific listeners (-l notls::) \n" #endif "-d, --daemon run as a daemon\n" "-r, --enable-coredumps maximize core file limit\n" "-u, --user= assume identity of (only when run as root)\n" "-m, --memory-limit= item memory in megabytes (default: %lu)\n" "-M, --disable-evictions return error on memory exhausted instead of evicting\n" "-c, --conn-limit= max simultaneous connections (default: %d)\n" "-k, --lock-memory lock down all paged memory\n" "-v, --verbose verbose (print errors/warnings while in event loop)\n" "-vv very verbose (also print client commands/responses)\n" "-vvv extremely verbose (internal state transitions)\n" "-h, --help print this help and exit\n" "-i, --license print memcached and libevent license\n" "-V, --version print version and exit\n" "-P, --pidfile= save PID in , only used with -d option\n" "-f, --slab-growth-factor= chunk size growth factor (default: %2.2f)\n" "-n, --slab-min-size= min space used for key+value+flags (default: %d)\n", settings.port, settings.udpport, settings.access, (unsigned long) settings.maxbytes / (1 << 20), settings.maxconns, settings.factor, settings.chunk_size); verify_default("udp-port",settings.udpport == 0); printf("-L, --enable-largepages try to use large memory pages (if available)\n"); printf("-D Use as the delimiter between key prefixes and IDs.\n" " This is used for per-prefix stats reporting. The default is\n" " \"%c\" (colon). If this option is specified, stats collection\n" " is turned on automatically; if not, then it may be turned on\n" " by sending the \"stats detail on\" command to the server.\n", settings.prefix_delimiter); printf("-t, --threads= number of threads to use (default: %d)\n", settings.num_threads); printf("-R, --max-reqs-per-event maximum number of requests per event, limits the\n" " requests processed per connection to prevent \n" " starvation (default: %d)\n", settings.reqs_per_event); printf("-C, --disable-cas disable use of CAS\n"); printf("-b, --listen-backlog= set the backlog queue limit (default: %d)\n", settings.backlog); printf("-B, --protocol= protocol - one of ascii, binary, or auto (default: %s)\n", prot_text(settings.binding_protocol)); printf("-I, --max-item-size= adjusts max item size\n" " (default: %dm, min: %dk, max: %dm)\n", settings.item_size_max/ (1 << 20), ITEM_SIZE_MAX_LOWER_LIMIT / (1 << 10), ITEM_SIZE_MAX_UPPER_LIMIT / (1 << 20)); #ifdef ENABLE_SASL printf("-S, --enable-sasl turn on Sasl authentication\n"); #endif printf("-F, --disable-flush-all disable flush_all command\n"); printf("-X, --disable-dumping disable stats cachedump and lru_crawler metadump\n"); printf("-W --disable-watch disable watch commands (live logging)\n"); printf("-Y, --auth-file= (EXPERIMENTAL) enable ASCII protocol authentication. format:\n" " user:pass\\nuser2:pass2\\n\n"); printf("-e, --memory-file= (EXPERIMENTAL) mmap a file for item memory.\n" " use only in ram disks or persistent memory mounts!\n" " enables restartable cache (stop with SIGUSR1)\n"); #ifdef TLS printf("-Z, --enable-ssl enable TLS/SSL\n"); #endif printf("-o, --extended comma separated list of extended options\n" " most options have a 'no_' prefix to disable\n" " - maxconns_fast: immediately close new connections after limit (default: %s)\n" " - hashpower: an integer multiplier for how large the hash\n" " table should be. normally grows at runtime. (default starts at: %d)\n" " set based on \"STAT hash_power_level\"\n" " - tail_repair_time: time in seconds for how long to wait before\n" " forcefully killing LRU tail item.\n" " disabled by default; very dangerous option.\n" " - hash_algorithm: the hash table algorithm\n" " default is murmur3 hash. options: jenkins, murmur3\n" " - no_lru_crawler: disable LRU Crawler background thread.\n" " - lru_crawler_sleep: microseconds to sleep between items\n" " default is %d.\n" " - lru_crawler_tocrawl: max items to crawl per slab per run\n" " default is %u (unlimited)\n", flag_enabled_disabled(settings.maxconns_fast), settings.hashpower_init, settings.lru_crawler_sleep, settings.lru_crawler_tocrawl); printf(" - no_lru_maintainer: disable new LRU system + background thread.\n" " - hot_lru_pct: pct of slab memory to reserve for hot lru.\n" " (requires lru_maintainer, default pct: %d)\n" " - warm_lru_pct: pct of slab memory to reserve for warm lru.\n" " (requires lru_maintainer, default pct: %d)\n" " - hot_max_factor: items idle > cold lru age * drop from hot lru. (default: %.2f)\n" " - warm_max_factor: items idle > cold lru age * this drop from warm. (default: %.2f)\n" " - temporary_ttl: TTL's below get separate LRU, can't be evicted.\n" " (requires lru_maintainer, default: %d)\n" " - idle_timeout: timeout for idle connections. (default: %d, no timeout)\n", settings.hot_lru_pct, settings.warm_lru_pct, settings.hot_max_factor, settings.warm_max_factor, settings.temporary_ttl, settings.idle_timeout); printf(" - slab_chunk_max: (EXPERIMENTAL) maximum slab size in kilobytes. use extreme care. (default: %d)\n" " - watcher_logbuf_size: size in kilobytes of per-watcher write buffer. (default: %u)\n" " - worker_logbuf_size: size in kilobytes of per-worker-thread buffer\n" " read by background thread, then written to watchers. (default: %u)\n" " - track_sizes: enable dynamic reports for 'stats sizes' command.\n" " - no_hashexpand: disables hash table expansion (dangerous)\n" " - modern: enables options which will be default in future.\n" " currently: nothing\n" " - no_modern: uses defaults of previous major version (1.4.x)\n", settings.slab_chunk_size_max / (1 << 10), settings.logger_watcher_buf_size / (1 << 10), settings.logger_buf_size / (1 << 10)); verify_default("tail_repair_time", settings.tail_repair_time == TAIL_REPAIR_TIME_DEFAULT); verify_default("lru_crawler_tocrawl", settings.lru_crawler_tocrawl == 0); verify_default("idle_timeout", settings.idle_timeout == 0); #ifdef HAVE_DROP_PRIVILEGES printf(" - drop_privileges: enable dropping extra syscall privileges\n" " - no_drop_privileges: disable drop_privileges in case it causes issues with\n" " some customisation.\n" " (default is no_drop_privileges)\n"); verify_default("drop_privileges", !settings.drop_privileges); #ifdef MEMCACHED_DEBUG printf(" - relaxed_privileges: running tests requires extra privileges. (default: %s)\n", flag_enabled_disabled(settings.relaxed_privileges)); #endif #endif #ifdef EXTSTORE printf(" - ext_path: file to write to for external storage.\n" " ie: ext_path=/mnt/d1/extstore:1G\n" " - ext_page_size: size in megabytes of storage pages. (default: %u)\n" " - ext_wbuf_size: size in megabytes of page write buffers. (default: %u)\n" " - ext_threads: number of IO threads to run. (default: %u)\n" " - ext_item_size: store items larger than this (bytes, default %u)\n" " - ext_item_age: store items idle at least this long (seconds, default: no age limit)\n" " - ext_low_ttl: consider TTLs lower than this specially (default: %u)\n" " - ext_drop_unread: don't re-write unread values during compaction (default: %s)\n" " - ext_recache_rate: recache an item every N accesses (default: %u)\n" " - ext_compact_under: compact when fewer than this many free pages\n" " (default: 1/4th of the assigned storage)\n" " - ext_drop_under: drop COLD items when fewer than this many free pages\n" " (default: 1/4th of the assigned storage)\n" " - ext_max_frag: max page fragmentation to tolerage (default: %.2f)\n" " - slab_automove_freeratio: ratio of memory to hold free as buffer.\n" " (see doc/storage.txt for more info, default: %.3f)\n", settings.ext_page_size / (1 << 20), settings.ext_wbuf_size / (1 << 20), settings.ext_io_threadcount, settings.ext_item_size, settings.ext_low_ttl, flag_enabled_disabled(settings.ext_drop_unread), settings.ext_recache_rate, settings.ext_max_frag, settings.slab_automove_freeratio); verify_default("ext_item_age", settings.ext_item_age == UINT_MAX); #endif #ifdef TLS printf(" - ssl_chain_cert: certificate chain file in PEM format\n" " - ssl_key: private key, if not part of the -ssl_chain_cert\n" " - ssl_keyformat: private key format (PEM, DER or ENGINE) (default: PEM)\n"); printf(" - ssl_verify_mode: peer certificate verification mode, default is 0(None).\n" " valid values are 0(None), 1(Request), 2(Require)\n" " or 3(Once)\n"); printf(" - ssl_ciphers: specify cipher list to be used\n" " - ssl_ca_cert: PEM format file of acceptable client CA's\n" " - ssl_wbuf_size: size in kilobytes of per-connection SSL output buffer\n" " (default: %u)\n", settings.ssl_wbuf_size / (1 << 10)); verify_default("ssl_keyformat", settings.ssl_keyformat == SSL_FILETYPE_PEM); verify_default("ssl_verify_mode", settings.ssl_verify_mode == SSL_VERIFY_NONE); #endif return; } static void usage_license(void) { printf(PACKAGE " " VERSION "\n\n"); printf( "Copyright (c) 2003, Danga Interactive, Inc. \n" "All rights reserved.\n" "\n" "Redistribution and use in source and binary forms, with or without\n" "modification, are permitted provided that the following conditions are\n" "met:\n" "\n" " * Redistributions of source code must retain the above copyright\n" "notice, this list of conditions and the following disclaimer.\n" "\n" " * Redistributions in binary form must reproduce the above\n" "copyright notice, this list of conditions and the following disclaimer\n" "in the documentation and/or other materials provided with the\n" "distribution.\n" "\n" " * Neither the name of the Danga Interactive nor the names of its\n" "contributors may be used to endorse or promote products derived from\n" "this software without specific prior written permission.\n" "\n" "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" "\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" "LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" "A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" "OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" "SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" "LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" "DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" "THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" "(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" "OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" "\n" "\n" "This product includes software developed by Niels Provos.\n" "\n" "[ libevent ]\n" "\n" "Copyright 2000-2003 Niels Provos \n" "All rights reserved.\n" "\n" "Redistribution and use in source and binary forms, with or without\n" "modification, are permitted provided that the following conditions\n" "are met:\n" "1. Redistributions of source code must retain the above copyright\n" " notice, this list of conditions and the following disclaimer.\n" "2. Redistributions in binary form must reproduce the above copyright\n" " notice, this list of conditions and the following disclaimer in the\n" " documentation and/or other materials provided with the distribution.\n" "3. All advertising materials mentioning features or use of this software\n" " must display the following acknowledgement:\n" " This product includes software developed by Niels Provos.\n" "4. The name of the author may not be used to endorse or promote products\n" " derived from this software without specific prior written permission.\n" "\n" "THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n" "IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n" "OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n" "IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n" "INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n" "NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" "DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" "THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" "(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n" "THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" ); return; } static void save_pid(const char *pid_file) { FILE *fp; if (access(pid_file, F_OK) == 0) { if ((fp = fopen(pid_file, "r")) != NULL) { char buffer[1024]; if (fgets(buffer, sizeof(buffer), fp) != NULL) { unsigned int pid; if (safe_strtoul(buffer, &pid) && kill((pid_t)pid, 0) == 0) { fprintf(stderr, "WARNING: The pid file contained the following (running) pid: %u\n", pid); } } fclose(fp); } } /* Create the pid file first with a temporary name, then * atomically move the file to the real name to avoid a race with * another process opening the file to read the pid, but finding * it empty. */ char tmp_pid_file[1024]; snprintf(tmp_pid_file, sizeof(tmp_pid_file), "%s.tmp", pid_file); if ((fp = fopen(tmp_pid_file, "w")) == NULL) { vperror("Could not open the pid file %s for writing", tmp_pid_file); return; } fprintf(fp,"%ld\n", (long)getpid()); if (fclose(fp) == -1) { vperror("Could not close the pid file %s", tmp_pid_file); } if (rename(tmp_pid_file, pid_file) != 0) { vperror("Could not rename the pid file from %s to %s", tmp_pid_file, pid_file); } } static void remove_pidfile(const char *pid_file) { if (pid_file == NULL) return; if (unlink(pid_file) != 0) { vperror("Could not remove the pid file %s", pid_file); } } static void sig_handler(const int sig) { printf("Signal handled: %s.\n", strsignal(sig)); exit(EXIT_SUCCESS); } static void sighup_handler(const int sig) { settings.sig_hup = true; } static void sig_usrhandler(const int sig) { printf("Graceful shutdown signal handled: %s.\n", strsignal(sig)); stop_main_loop = true; } #ifndef HAVE_SIGIGNORE static int sigignore(int sig) { struct sigaction sa = { .sa_handler = SIG_IGN, .sa_flags = 0 }; if (sigemptyset(&sa.sa_mask) == -1 || sigaction(sig, &sa, 0) == -1) { return -1; } return 0; } #endif /* * On systems that supports multiple page sizes we may reduce the * number of TLB-misses by using the biggest available page size */ static int enable_large_pages(void) { #if defined(HAVE_GETPAGESIZES) && defined(HAVE_MEMCNTL) int ret = -1; size_t sizes[32]; int avail = getpagesizes(sizes, 32); if (avail != -1) { size_t max = sizes[0]; struct memcntl_mha arg = {0}; int ii; for (ii = 1; ii < avail; ++ii) { if (max < sizes[ii]) { max = sizes[ii]; } } arg.mha_flags = 0; arg.mha_pagesize = max; arg.mha_cmd = MHA_MAPSIZE_BSSBRK; if (memcntl(0, 0, MC_HAT_ADVISE, (caddr_t)&arg, 0, 0) == -1) { fprintf(stderr, "Failed to set large pages: %s\n", strerror(errno)); fprintf(stderr, "Will use default page size\n"); } else { ret = 0; } } else { fprintf(stderr, "Failed to get supported pagesizes: %s\n", strerror(errno)); fprintf(stderr, "Will use default page size\n"); } return ret; #elif defined(__linux__) && defined(MADV_HUGEPAGE) /* check if transparent hugepages is compiled into the kernel */ struct stat st; int ret = stat("/sys/kernel/mm/transparent_hugepage/enabled", &st); if (ret || !(st.st_mode & S_IFREG)) { fprintf(stderr, "Transparent huge pages support not detected.\n"); fprintf(stderr, "Will use default page size.\n"); return -1; } return 0; #elif defined(__FreeBSD__) int spages; size_t spagesl = sizeof(spages); if (sysctlbyname("vm.pmap.pg_ps_enabled", &spages, &spagesl, NULL, 0) != 0) { fprintf(stderr, "Could not evaluate the presence of superpages features."); return -1; } if (spages != 1) { fprintf(stderr, "Superpages support not detected.\n"); fprintf(stderr, "Will use default page size.\n"); return -1; } return 0; #else return -1; #endif } /** * Do basic sanity check of the runtime environment * @return true if no errors found, false if we can't use this env */ static bool sanitycheck(void) { /* One of our biggest problems is old and bogus libevents */ const char *ever = event_get_version(); if (ever != NULL) { if (strncmp(ever, "1.", 2) == 0) { /* Require at least 1.3 (that's still a couple of years old) */ if (('0' <= ever[2] && ever[2] < '3') && !isdigit(ever[3])) { fprintf(stderr, "You are using libevent %s.\nPlease upgrade to" " a more recent version (1.3 or newer)\n", event_get_version()); return false; } } } return true; } static bool _parse_slab_sizes(char *s, uint32_t *slab_sizes) { char *b = NULL; uint32_t size = 0; int i = 0; uint32_t last_size = 0; if (strlen(s) < 1) return false; for (char *p = strtok_r(s, "-", &b); p != NULL; p = strtok_r(NULL, "-", &b)) { if (!safe_strtoul(p, &size) || size < settings.chunk_size || size > settings.slab_chunk_size_max) { fprintf(stderr, "slab size %u is out of valid range\n", size); return false; } if (last_size >= size) { fprintf(stderr, "slab size %u cannot be lower than or equal to a previous class size\n", size); return false; } if (size <= last_size + CHUNK_ALIGN_BYTES) { fprintf(stderr, "slab size %u must be at least %d bytes larger than previous class\n", size, CHUNK_ALIGN_BYTES); return false; } slab_sizes[i++] = size; last_size = size; if (i >= MAX_NUMBER_OF_SLAB_CLASSES-1) { fprintf(stderr, "too many slab classes specified\n"); return false; } } slab_sizes[i] = 0; return true; } struct _mc_meta_data { void *mmap_base; uint64_t old_base; char *slab_config; // string containing either factor or custom slab list. int64_t time_delta; uint64_t process_started; uint32_t current_time; }; // We need to remember a combination of configuration settings and global // state for restart viability and resumption of internal services. // Compared to the number of tunables and state values, relatively little // does need to be remembered. // Time is the hardest; we have to assume the sys clock is correct and re-sync for // the lost time after restart. static int _mc_meta_save_cb(const char *tag, void *ctx, void *data) { struct _mc_meta_data *meta = (struct _mc_meta_data *)data; // Settings to remember. // TODO: should get a version of version which is numeric, else // comparisons for compat reasons are difficult. // it may be possible to punt on this for now; since we can test for the // absense of another key... such as the new numeric version. //restart_set_kv(ctx, "version", "%s", VERSION); // We hold the original factor or subopts _string_ // it can be directly compared without roundtripping through floats or // serializing/deserializing the long options list. restart_set_kv(ctx, "slab_config", "%s", meta->slab_config); restart_set_kv(ctx, "maxbytes", "%llu", (unsigned long long) settings.maxbytes); restart_set_kv(ctx, "chunk_size", "%d", settings.chunk_size); restart_set_kv(ctx, "item_size_max", "%d", settings.item_size_max); restart_set_kv(ctx, "slab_chunk_size_max", "%d", settings.slab_chunk_size_max); restart_set_kv(ctx, "slab_page_size", "%d", settings.slab_page_size); restart_set_kv(ctx, "use_cas", "%s", settings.use_cas ? "true" : "false"); restart_set_kv(ctx, "slab_reassign", "%s", settings.slab_reassign ? "true" : "false"); // Online state to remember. // current time is tough. we need to rely on the clock being correct to // pull the delta between stop and start times. we also need to know the // delta between start time and now to restore monotonic clocks. // for non-monotonic clocks (some OS?), process_started is the only // important one. restart_set_kv(ctx, "current_time", "%u", current_time); // types are great until... this. some systems time_t could be big, but // I'm assuming never negative. restart_set_kv(ctx, "process_started", "%llu", (unsigned long long) process_started); { struct timeval tv; gettimeofday(&tv, NULL); restart_set_kv(ctx, "stop_time", "%lu", tv.tv_sec); } // Might as well just fetch the next CAS value to use than tightly // coupling the internal variable into the restart system. restart_set_kv(ctx, "current_cas", "%llu", (unsigned long long) get_cas_id()); restart_set_kv(ctx, "oldest_cas", "%llu", (unsigned long long) settings.oldest_cas); restart_set_kv(ctx, "logger_gid", "%llu", logger_get_gid()); restart_set_kv(ctx, "hashpower", "%u", stats_state.hash_power_level); // NOTE: oldest_live is a rel_time_t, which aliases for unsigned int. // should future proof this with a 64bit upcast, or fetch value from a // converter function/macro? restart_set_kv(ctx, "oldest_live", "%u", settings.oldest_live); // TODO: use uintptr_t etc? is it portable enough? restart_set_kv(ctx, "mmap_oldbase", "%p", meta->mmap_base); return 0; } // We must see at least this number of checked lines. Else empty/missing lines // could cause a false-positive. // TODO: Once crc32'ing of the metadata file is done this could be ensured better by // the restart module itself (crc32 + count of lines must match on the // backend) #define RESTART_REQUIRED_META 17 // With this callback we make a decision on if the current configuration // matches up enough to allow reusing the cache. // We also re-load important runtime information. static int _mc_meta_load_cb(const char *tag, void *ctx, void *data) { struct _mc_meta_data *meta = (struct _mc_meta_data *)data; char *key; char *val; int reuse_mmap = 0; meta->process_started = 0; meta->time_delta = 0; meta->current_time = 0; int lines_seen = 0; // TODO: not sure this is any better than just doing an if/else tree with // strcmp's... enum { R_MMAP_OLDBASE = 0, R_MAXBYTES, R_CHUNK_SIZE, R_ITEM_SIZE_MAX, R_SLAB_CHUNK_SIZE_MAX, R_SLAB_PAGE_SIZE, R_SLAB_CONFIG, R_USE_CAS, R_SLAB_REASSIGN, R_CURRENT_CAS, R_OLDEST_CAS, R_OLDEST_LIVE, R_LOGGER_GID, R_CURRENT_TIME, R_STOP_TIME, R_PROCESS_STARTED, R_HASHPOWER, }; const char *opts[] = { [R_MMAP_OLDBASE] = "mmap_oldbase", [R_MAXBYTES] = "maxbytes", [R_CHUNK_SIZE] = "chunk_size", [R_ITEM_SIZE_MAX] = "item_size_max", [R_SLAB_CHUNK_SIZE_MAX] = "slab_chunk_size_max", [R_SLAB_PAGE_SIZE] = "slab_page_size", [R_SLAB_CONFIG] = "slab_config", [R_USE_CAS] = "use_cas", [R_SLAB_REASSIGN] = "slab_reassign", [R_CURRENT_CAS] = "current_cas", [R_OLDEST_CAS] = "oldest_cas", [R_OLDEST_LIVE] = "oldest_live", [R_LOGGER_GID] = "logger_gid", [R_CURRENT_TIME] = "current_time", [R_STOP_TIME] = "stop_time", [R_PROCESS_STARTED] = "process_started", [R_HASHPOWER] = "hashpower", NULL }; while (restart_get_kv(ctx, &key, &val) == RESTART_OK) { int type = 0; int32_t val_int = 0; uint32_t val_uint = 0; int64_t bigval_int = 0; uint64_t bigval_uint = 0; while (opts[type] != NULL && strcmp(key, opts[type]) != 0) { type++; } if (opts[type] == NULL) { fprintf(stderr, "[restart] unknown/unhandled key: %s\n", key); continue; } lines_seen++; // helper for any boolean checkers. bool val_bool = false; bool is_bool = true; if (strcmp(val, "false") == 0) { val_bool = false; } else if (strcmp(val, "true") == 0) { val_bool = true; } else { is_bool = false; } switch (type) { case R_MMAP_OLDBASE: if (!safe_strtoull_hex(val, &meta->old_base)) { fprintf(stderr, "[restart] failed to parse %s: %s\n", key, val); reuse_mmap = -1; } break; case R_MAXBYTES: if (!safe_strtoll(val, &bigval_int) || settings.maxbytes != bigval_int) { reuse_mmap = -1; } break; case R_CHUNK_SIZE: if (!safe_strtol(val, &val_int) || settings.chunk_size != val_int) { reuse_mmap = -1; } break; case R_ITEM_SIZE_MAX: if (!safe_strtol(val, &val_int) || settings.item_size_max != val_int) { reuse_mmap = -1; } break; case R_SLAB_CHUNK_SIZE_MAX: if (!safe_strtol(val, &val_int) || settings.slab_chunk_size_max != val_int) { reuse_mmap = -1; } break; case R_SLAB_PAGE_SIZE: if (!safe_strtol(val, &val_int) || settings.slab_page_size != val_int) { reuse_mmap = -1; } break; case R_SLAB_CONFIG: if (strcmp(val, meta->slab_config) != 0) { reuse_mmap = -1; } break; case R_USE_CAS: if (!is_bool || settings.use_cas != val_bool) { reuse_mmap = -1; } break; case R_SLAB_REASSIGN: if (!is_bool || settings.slab_reassign != val_bool) { reuse_mmap = -1; } break; case R_CURRENT_CAS: // FIXME: do we need to fail if these values _aren't_ found? if (!safe_strtoull(val, &bigval_uint)) { reuse_mmap = -1; } else { set_cas_id(bigval_uint); } break; case R_OLDEST_CAS: if (!safe_strtoull(val, &bigval_uint)) { reuse_mmap = -1; } else { settings.oldest_cas = bigval_uint; } break; case R_OLDEST_LIVE: if (!safe_strtoul(val, &val_uint)) { reuse_mmap = -1; } else { settings.oldest_live = val_uint; } break; case R_LOGGER_GID: if (!safe_strtoull(val, &bigval_uint)) { reuse_mmap = -1; } else { logger_set_gid(bigval_uint); } break; case R_PROCESS_STARTED: if (!safe_strtoull(val, &bigval_uint)) { reuse_mmap = -1; } else { meta->process_started = bigval_uint; } break; case R_CURRENT_TIME: if (!safe_strtoul(val, &val_uint)) { reuse_mmap = -1; } else { meta->current_time = val_uint; } break; case R_STOP_TIME: if (!safe_strtoll(val, &bigval_int)) { reuse_mmap = -1; } else { struct timeval t; gettimeofday(&t, NULL); meta->time_delta = t.tv_sec - bigval_int; // clock has done something crazy. // there are _lots_ of ways the clock can go wrong here, but // this is a safe sanity check since there's nothing else we // can realistically do. if (meta->time_delta <= 0) { reuse_mmap = -1; } } break; case R_HASHPOWER: if (!safe_strtoul(val, &val_uint)) { reuse_mmap = -1; } else { settings.hashpower_init = val_uint; } break; default: fprintf(stderr, "[restart] unhandled key: %s\n", key); } if (reuse_mmap != 0) { fprintf(stderr, "[restart] restart incompatible due to setting for [%s] [old value: %s]\n", key, val); break; } } if (lines_seen < RESTART_REQUIRED_META) { fprintf(stderr, "[restart] missing some metadata lines\n"); reuse_mmap = -1; } return reuse_mmap; } int main (int argc, char **argv) { int c; bool lock_memory = false; bool do_daemonize = false; bool preallocate = false; int maxcore = 0; char *username = NULL; char *pid_file = NULL; char *memory_file = NULL; struct passwd *pw; struct rlimit rlim; char *buf; char unit = '\0'; int size_max = 0; int retval = EXIT_SUCCESS; bool protocol_specified = false; bool tcp_specified = false; bool udp_specified = false; bool start_lru_maintainer = true; bool start_lru_crawler = true; bool start_assoc_maint = true; enum hashfunc_type hash_type = MURMUR3_HASH; uint32_t tocrawl; uint32_t slab_sizes[MAX_NUMBER_OF_SLAB_CLASSES]; bool use_slab_sizes = false; char *slab_sizes_unparsed = NULL; bool slab_chunk_size_changed = false; // struct for restart code. Initialized up here so we can curry // important settings to save or validate. struct _mc_meta_data *meta = malloc(sizeof(struct _mc_meta_data)); meta->slab_config = NULL; #ifdef EXTSTORE void *storage = NULL; struct extstore_conf_file *storage_file = NULL; struct extstore_conf ext_cf; #endif char *subopts, *subopts_orig; char *subopts_value; enum { MAXCONNS_FAST = 0, HASHPOWER_INIT, NO_HASHEXPAND, SLAB_REASSIGN, SLAB_AUTOMOVE, SLAB_AUTOMOVE_RATIO, SLAB_AUTOMOVE_WINDOW, TAIL_REPAIR_TIME, HASH_ALGORITHM, LRU_CRAWLER, LRU_CRAWLER_SLEEP, LRU_CRAWLER_TOCRAWL, LRU_MAINTAINER, HOT_LRU_PCT, WARM_LRU_PCT, HOT_MAX_FACTOR, WARM_MAX_FACTOR, TEMPORARY_TTL, IDLE_TIMEOUT, WATCHER_LOGBUF_SIZE, WORKER_LOGBUF_SIZE, SLAB_SIZES, SLAB_CHUNK_MAX, TRACK_SIZES, NO_INLINE_ASCII_RESP, MODERN, NO_MODERN, NO_CHUNKED_ITEMS, NO_SLAB_REASSIGN, NO_SLAB_AUTOMOVE, NO_MAXCONNS_FAST, INLINE_ASCII_RESP, NO_LRU_CRAWLER, NO_LRU_MAINTAINER, NO_DROP_PRIVILEGES, DROP_PRIVILEGES, #ifdef TLS SSL_CERT, SSL_KEY, SSL_VERIFY_MODE, SSL_KEYFORM, SSL_CIPHERS, SSL_CA_CERT, SSL_WBUF_SIZE, #endif #ifdef MEMCACHED_DEBUG RELAXED_PRIVILEGES, #endif #ifdef EXTSTORE EXT_PAGE_SIZE, EXT_WBUF_SIZE, EXT_THREADS, EXT_IO_DEPTH, EXT_PATH, EXT_ITEM_SIZE, EXT_ITEM_AGE, EXT_LOW_TTL, EXT_RECACHE_RATE, EXT_COMPACT_UNDER, EXT_DROP_UNDER, EXT_MAX_FRAG, EXT_DROP_UNREAD, SLAB_AUTOMOVE_FREERATIO, #endif }; char *const subopts_tokens[] = { [MAXCONNS_FAST] = "maxconns_fast", [HASHPOWER_INIT] = "hashpower", [NO_HASHEXPAND] = "no_hashexpand", [SLAB_REASSIGN] = "slab_reassign", [SLAB_AUTOMOVE] = "slab_automove", [SLAB_AUTOMOVE_RATIO] = "slab_automove_ratio", [SLAB_AUTOMOVE_WINDOW] = "slab_automove_window", [TAIL_REPAIR_TIME] = "tail_repair_time", [HASH_ALGORITHM] = "hash_algorithm", [LRU_CRAWLER] = "lru_crawler", [LRU_CRAWLER_SLEEP] = "lru_crawler_sleep", [LRU_CRAWLER_TOCRAWL] = "lru_crawler_tocrawl", [LRU_MAINTAINER] = "lru_maintainer", [HOT_LRU_PCT] = "hot_lru_pct", [WARM_LRU_PCT] = "warm_lru_pct", [HOT_MAX_FACTOR] = "hot_max_factor", [WARM_MAX_FACTOR] = "warm_max_factor", [TEMPORARY_TTL] = "temporary_ttl", [IDLE_TIMEOUT] = "idle_timeout", [WATCHER_LOGBUF_SIZE] = "watcher_logbuf_size", [WORKER_LOGBUF_SIZE] = "worker_logbuf_size", [SLAB_SIZES] = "slab_sizes", [SLAB_CHUNK_MAX] = "slab_chunk_max", [TRACK_SIZES] = "track_sizes", [NO_INLINE_ASCII_RESP] = "no_inline_ascii_resp", [MODERN] = "modern", [NO_MODERN] = "no_modern", [NO_CHUNKED_ITEMS] = "no_chunked_items", [NO_SLAB_REASSIGN] = "no_slab_reassign", [NO_SLAB_AUTOMOVE] = "no_slab_automove", [NO_MAXCONNS_FAST] = "no_maxconns_fast", [INLINE_ASCII_RESP] = "inline_ascii_resp", [NO_LRU_CRAWLER] = "no_lru_crawler", [NO_LRU_MAINTAINER] = "no_lru_maintainer", [NO_DROP_PRIVILEGES] = "no_drop_privileges", [DROP_PRIVILEGES] = "drop_privileges", #ifdef TLS [SSL_CERT] = "ssl_chain_cert", [SSL_KEY] = "ssl_key", [SSL_VERIFY_MODE] = "ssl_verify_mode", [SSL_KEYFORM] = "ssl_keyformat", [SSL_CIPHERS] = "ssl_ciphers", [SSL_CA_CERT] = "ssl_ca_cert", [SSL_WBUF_SIZE] = "ssl_wbuf_size", #endif #ifdef MEMCACHED_DEBUG [RELAXED_PRIVILEGES] = "relaxed_privileges", #endif #ifdef EXTSTORE [EXT_PAGE_SIZE] = "ext_page_size", [EXT_WBUF_SIZE] = "ext_wbuf_size", [EXT_THREADS] = "ext_threads", [EXT_IO_DEPTH] = "ext_io_depth", [EXT_PATH] = "ext_path", [EXT_ITEM_SIZE] = "ext_item_size", [EXT_ITEM_AGE] = "ext_item_age", [EXT_LOW_TTL] = "ext_low_ttl", [EXT_RECACHE_RATE] = "ext_recache_rate", [EXT_COMPACT_UNDER] = "ext_compact_under", [EXT_DROP_UNDER] = "ext_drop_under", [EXT_MAX_FRAG] = "ext_max_frag", [EXT_DROP_UNREAD] = "ext_drop_unread", [SLAB_AUTOMOVE_FREERATIO] = "slab_automove_freeratio", #endif NULL }; if (!sanitycheck()) { free(meta); return EX_OSERR; } /* handle SIGINT, SIGTERM */ signal(SIGINT, sig_handler); signal(SIGTERM, sig_handler); signal(SIGHUP, sighup_handler); signal(SIGUSR1, sig_usrhandler); /* init settings */ settings_init(); verify_default("hash_algorithm", hash_type == MURMUR3_HASH); #ifdef EXTSTORE settings.ext_item_size = 512; settings.ext_item_age = UINT_MAX; settings.ext_low_ttl = 0; settings.ext_recache_rate = 2000; settings.ext_max_frag = 0.8; settings.ext_drop_unread = false; settings.ext_wbuf_size = 1024 * 1024 * 4; settings.ext_compact_under = 0; settings.ext_drop_under = 0; settings.slab_automove_freeratio = 0.01; settings.ext_page_size = 1024 * 1024 * 64; settings.ext_io_threadcount = 1; ext_cf.page_size = settings.ext_page_size; ext_cf.wbuf_size = settings.ext_wbuf_size; ext_cf.io_threadcount = settings.ext_io_threadcount; ext_cf.io_depth = 1; ext_cf.page_buckets = 4; ext_cf.wbuf_count = ext_cf.page_buckets; #endif /* Run regardless of initializing it later */ init_lru_maintainer(); /* set stderr non-buffering (for running under, say, daemontools) */ setbuf(stderr, NULL); char *shortopts = "a:" /* access mask for unix socket */ "A" /* enable admin shutdown command */ "Z" /* enable SSL */ "p:" /* TCP port number to listen on */ "s:" /* unix socket path to listen on */ "U:" /* UDP port number to listen on */ "m:" /* max memory to use for items in megabytes */ "M" /* return error on memory exhausted */ "c:" /* max simultaneous connections */ "k" /* lock down all paged memory */ "hiV" /* help, licence info, version */ "r" /* maximize core file limit */ "v" /* verbose */ "d" /* daemon mode */ "l:" /* interface to listen on */ "u:" /* user identity to run as */ "P:" /* save PID in file */ "f:" /* factor? */ "n:" /* minimum space allocated for key+value+flags */ "t:" /* threads */ "D:" /* prefix delimiter? */ "L" /* Large memory pages */ "R:" /* max requests per event */ "C" /* Disable use of CAS */ "b:" /* backlog queue limit */ "B:" /* Binding protocol */ "I:" /* Max item size */ "S" /* Sasl ON */ "F" /* Disable flush_all */ "X" /* Disable dump commands */ "W" /* Disable watch commands */ "Y:" /* Enable token auth */ "e:" /* mmap path for external item memory */ "o:" /* Extended generic options */ ; /* process arguments */ #ifdef HAVE_GETOPT_LONG const struct option longopts[] = { {"unix-mask", required_argument, 0, 'a'}, {"enable-shutdown", no_argument, 0, 'A'}, {"enable-ssl", no_argument, 0, 'Z'}, {"port", required_argument, 0, 'p'}, {"unix-socket", required_argument, 0, 's'}, {"udp-port", required_argument, 0, 'U'}, {"memory-limit", required_argument, 0, 'm'}, {"disable-evictions", no_argument, 0, 'M'}, {"conn-limit", required_argument, 0, 'c'}, {"lock-memory", no_argument, 0, 'k'}, {"help", no_argument, 0, 'h'}, {"license", no_argument, 0, 'i'}, {"version", no_argument, 0, 'V'}, {"enable-coredumps", no_argument, 0, 'r'}, {"verbose", optional_argument, 0, 'v'}, {"daemon", no_argument, 0, 'd'}, {"listen", required_argument, 0, 'l'}, {"user", required_argument, 0, 'u'}, {"pidfile", required_argument, 0, 'P'}, {"slab-growth-factor", required_argument, 0, 'f'}, {"slab-min-size", required_argument, 0, 'n'}, {"threads", required_argument, 0, 't'}, {"enable-largepages", no_argument, 0, 'L'}, {"max-reqs-per-event", required_argument, 0, 'R'}, {"disable-cas", no_argument, 0, 'C'}, {"listen-backlog", required_argument, 0, 'b'}, {"protocol", required_argument, 0, 'B'}, {"max-item-size", required_argument, 0, 'I'}, {"enable-sasl", no_argument, 0, 'S'}, {"disable-flush-all", no_argument, 0, 'F'}, {"disable-dumping", no_argument, 0, 'X'}, {"disable-watch", no_argument, 0, 'W'}, {"auth-file", required_argument, 0, 'Y'}, {"memory-file", required_argument, 0, 'e'}, {"extended", required_argument, 0, 'o'}, {0, 0, 0, 0} }; int optindex; while (-1 != (c = getopt_long(argc, argv, shortopts, longopts, &optindex))) { #else while (-1 != (c = getopt(argc, argv, shortopts))) { #endif switch (c) { case 'A': /* enables "shutdown" command */ settings.shutdown_command = true; break; case 'Z': /* enable secure communication*/ #ifdef TLS settings.ssl_enabled = true; #else fprintf(stderr, "This server is not built with TLS support.\n"); exit(EX_USAGE); #endif break; case 'a': /* access for unix domain socket, as octal mask (like chmod)*/ settings.access= strtol(optarg,NULL,8); break; case 'U': settings.udpport = atoi(optarg); udp_specified = true; break; case 'p': settings.port = atoi(optarg); tcp_specified = true; break; case 's': settings.socketpath = optarg; break; case 'm': settings.maxbytes = ((size_t)atoi(optarg)) * 1024 * 1024; break; case 'M': settings.evict_to_free = 0; break; case 'c': settings.maxconns = atoi(optarg); if (settings.maxconns <= 0) { fprintf(stderr, "Maximum connections must be greater than 0\n"); return 1; } break; case 'h': usage(); exit(EXIT_SUCCESS); case 'i': usage_license(); exit(EXIT_SUCCESS); case 'V': printf(PACKAGE " " VERSION "\n"); exit(EXIT_SUCCESS); case 'k': lock_memory = true; break; case 'v': settings.verbose++; break; case 'l': if (settings.inter != NULL) { if (strstr(settings.inter, optarg) != NULL) { break; } size_t len = strlen(settings.inter) + strlen(optarg) + 2; char *p = malloc(len); if (p == NULL) { fprintf(stderr, "Failed to allocate memory\n"); return 1; } snprintf(p, len, "%s,%s", settings.inter, optarg); free(settings.inter); settings.inter = p; } else { settings.inter= strdup(optarg); } break; case 'd': do_daemonize = true; break; case 'r': maxcore = 1; break; case 'R': settings.reqs_per_event = atoi(optarg); if (settings.reqs_per_event == 0) { fprintf(stderr, "Number of requests per event must be greater than 0\n"); return 1; } break; case 'u': username = optarg; break; case 'P': pid_file = optarg; break; case 'e': memory_file = optarg; break; case 'f': settings.factor = atof(optarg); if (settings.factor <= 1.0) { fprintf(stderr, "Factor must be greater than 1\n"); return 1; } meta->slab_config = strdup(optarg); break; case 'n': settings.chunk_size = atoi(optarg); if (settings.chunk_size == 0) { fprintf(stderr, "Chunk size must be greater than 0\n"); return 1; } break; case 't': settings.num_threads = atoi(optarg); if (settings.num_threads <= 0) { fprintf(stderr, "Number of threads must be greater than 0\n"); return 1; } /* There're other problems when you get above 64 threads. * In the future we should portably detect # of cores for the * default. */ if (settings.num_threads > 64) { fprintf(stderr, "WARNING: Setting a high number of worker" "threads is not recommended.\n" " Set this value to the number of cores in" " your machine or less.\n"); } break; case 'D': if (! optarg || ! optarg[0]) { fprintf(stderr, "No delimiter specified\n"); return 1; } settings.prefix_delimiter = optarg[0]; settings.detail_enabled = 1; break; case 'L' : if (enable_large_pages() == 0) { preallocate = true; } else { fprintf(stderr, "Cannot enable large pages on this system\n" "(There is no support as of this version)\n"); return 1; } break; case 'C' : settings.use_cas = false; break; case 'b' : settings.backlog = atoi(optarg); break; case 'B': protocol_specified = true; if (strcmp(optarg, "auto") == 0) { settings.binding_protocol = negotiating_prot; } else if (strcmp(optarg, "binary") == 0) { settings.binding_protocol = binary_prot; } else if (strcmp(optarg, "ascii") == 0) { settings.binding_protocol = ascii_prot; } else { fprintf(stderr, "Invalid value for binding protocol: %s\n" " -- should be one of auto, binary, or ascii\n", optarg); exit(EX_USAGE); } break; case 'I': buf = strdup(optarg); unit = buf[strlen(buf)-1]; if (unit == 'k' || unit == 'm' || unit == 'K' || unit == 'M') { buf[strlen(buf)-1] = '\0'; size_max = atoi(buf); if (unit == 'k' || unit == 'K') size_max *= 1024; if (unit == 'm' || unit == 'M') size_max *= 1024 * 1024; settings.item_size_max = size_max; } else { settings.item_size_max = atoi(buf); } free(buf); break; case 'S': /* set Sasl authentication to true. Default is false */ #ifndef ENABLE_SASL fprintf(stderr, "This server is not built with SASL support.\n"); exit(EX_USAGE); #endif settings.sasl = true; break; case 'F' : settings.flush_enabled = false; break; case 'X' : settings.dump_enabled = false; break; case 'W' : settings.watch_enabled = false; break; case 'Y' : // dupe the file path now just in case the options get mangled. settings.auth_file = strdup(optarg); break; case 'o': /* It's sub-opts time! */ subopts_orig = subopts = strdup(optarg); /* getsubopt() changes the original args */ while (*subopts != '\0') { switch (getsubopt(&subopts, subopts_tokens, &subopts_value)) { case MAXCONNS_FAST: settings.maxconns_fast = true; break; case HASHPOWER_INIT: if (subopts_value == NULL) { fprintf(stderr, "Missing numeric argument for hashpower\n"); return 1; } settings.hashpower_init = atoi(subopts_value); if (settings.hashpower_init < 12) { fprintf(stderr, "Initial hashtable multiplier of %d is too low\n", settings.hashpower_init); return 1; } else if (settings.hashpower_init > 32) { fprintf(stderr, "Initial hashtable multiplier of %d is too high\n" "Choose a value based on \"STAT hash_power_level\" from a running instance\n", settings.hashpower_init); return 1; } break; case NO_HASHEXPAND: start_assoc_maint = false; break; case SLAB_REASSIGN: settings.slab_reassign = true; break; case SLAB_AUTOMOVE: if (subopts_value == NULL) { settings.slab_automove = 1; break; } settings.slab_automove = atoi(subopts_value); if (settings.slab_automove < 0 || settings.slab_automove > 2) { fprintf(stderr, "slab_automove must be between 0 and 2\n"); return 1; } break; case SLAB_AUTOMOVE_RATIO: if (subopts_value == NULL) { fprintf(stderr, "Missing slab_automove_ratio argument\n"); return 1; } settings.slab_automove_ratio = atof(subopts_value); if (settings.slab_automove_ratio <= 0 || settings.slab_automove_ratio > 1) { fprintf(stderr, "slab_automove_ratio must be > 0 and < 1\n"); return 1; } break; case SLAB_AUTOMOVE_WINDOW: if (subopts_value == NULL) { fprintf(stderr, "Missing slab_automove_window argument\n"); return 1; } settings.slab_automove_window = atoi(subopts_value); if (settings.slab_automove_window < 3) { fprintf(stderr, "slab_automove_window must be > 2\n"); return 1; } break; case TAIL_REPAIR_TIME: if (subopts_value == NULL) { fprintf(stderr, "Missing numeric argument for tail_repair_time\n"); return 1; } settings.tail_repair_time = atoi(subopts_value); if (settings.tail_repair_time < 10) { fprintf(stderr, "Cannot set tail_repair_time to less than 10 seconds\n"); return 1; } break; case HASH_ALGORITHM: if (subopts_value == NULL) { fprintf(stderr, "Missing hash_algorithm argument\n"); return 1; }; if (strcmp(subopts_value, "jenkins") == 0) { hash_type = JENKINS_HASH; } else if (strcmp(subopts_value, "murmur3") == 0) { hash_type = MURMUR3_HASH; } else { fprintf(stderr, "Unknown hash_algorithm option (jenkins, murmur3)\n"); return 1; } break; case LRU_CRAWLER: start_lru_crawler = true; break; case LRU_CRAWLER_SLEEP: if (subopts_value == NULL) { fprintf(stderr, "Missing lru_crawler_sleep value\n"); return 1; } settings.lru_crawler_sleep = atoi(subopts_value); if (settings.lru_crawler_sleep > 1000000 || settings.lru_crawler_sleep < 0) { fprintf(stderr, "LRU crawler sleep must be between 0 and 1 second\n"); return 1; } break; case LRU_CRAWLER_TOCRAWL: if (subopts_value == NULL) { fprintf(stderr, "Missing lru_crawler_tocrawl value\n"); return 1; } if (!safe_strtoul(subopts_value, &tocrawl)) { fprintf(stderr, "lru_crawler_tocrawl takes a numeric 32bit value\n"); return 1; } settings.lru_crawler_tocrawl = tocrawl; break; case LRU_MAINTAINER: start_lru_maintainer = true; settings.lru_segmented = true; break; case HOT_LRU_PCT: if (subopts_value == NULL) { fprintf(stderr, "Missing hot_lru_pct argument\n"); return 1; } settings.hot_lru_pct = atoi(subopts_value); if (settings.hot_lru_pct < 1 || settings.hot_lru_pct >= 80) { fprintf(stderr, "hot_lru_pct must be > 1 and < 80\n"); return 1; } break; case WARM_LRU_PCT: if (subopts_value == NULL) { fprintf(stderr, "Missing warm_lru_pct argument\n"); return 1; } settings.warm_lru_pct = atoi(subopts_value); if (settings.warm_lru_pct < 1 || settings.warm_lru_pct >= 80) { fprintf(stderr, "warm_lru_pct must be > 1 and < 80\n"); return 1; } break; case HOT_MAX_FACTOR: if (subopts_value == NULL) { fprintf(stderr, "Missing hot_max_factor argument\n"); return 1; } settings.hot_max_factor = atof(subopts_value); if (settings.hot_max_factor <= 0) { fprintf(stderr, "hot_max_factor must be > 0\n"); return 1; } break; case WARM_MAX_FACTOR: if (subopts_value == NULL) { fprintf(stderr, "Missing warm_max_factor argument\n"); return 1; } settings.warm_max_factor = atof(subopts_value); if (settings.warm_max_factor <= 0) { fprintf(stderr, "warm_max_factor must be > 0\n"); return 1; } break; case TEMPORARY_TTL: if (subopts_value == NULL) { fprintf(stderr, "Missing temporary_ttl argument\n"); return 1; } settings.temp_lru = true; settings.temporary_ttl = atoi(subopts_value); break; case IDLE_TIMEOUT: if (subopts_value == NULL) { fprintf(stderr, "Missing numeric argument for idle_timeout\n"); return 1; } settings.idle_timeout = atoi(subopts_value); break; case WATCHER_LOGBUF_SIZE: if (subopts_value == NULL) { fprintf(stderr, "Missing watcher_logbuf_size argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.logger_watcher_buf_size)) { fprintf(stderr, "could not parse argument to watcher_logbuf_size\n"); return 1; } settings.logger_watcher_buf_size *= 1024; /* kilobytes */ break; case WORKER_LOGBUF_SIZE: if (subopts_value == NULL) { fprintf(stderr, "Missing worker_logbuf_size argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.logger_buf_size)) { fprintf(stderr, "could not parse argument to worker_logbuf_size\n"); return 1; } settings.logger_buf_size *= 1024; /* kilobytes */ case SLAB_SIZES: slab_sizes_unparsed = strdup(subopts_value); break; case SLAB_CHUNK_MAX: if (subopts_value == NULL) { fprintf(stderr, "Missing slab_chunk_max argument\n"); } if (!safe_strtol(subopts_value, &settings.slab_chunk_size_max)) { fprintf(stderr, "could not parse argument to slab_chunk_max\n"); } slab_chunk_size_changed = true; break; case TRACK_SIZES: item_stats_sizes_init(); break; case NO_INLINE_ASCII_RESP: break; case INLINE_ASCII_RESP: break; case NO_CHUNKED_ITEMS: settings.slab_chunk_size_max = settings.slab_page_size; break; case NO_SLAB_REASSIGN: settings.slab_reassign = false; break; case NO_SLAB_AUTOMOVE: settings.slab_automove = 0; break; case NO_MAXCONNS_FAST: settings.maxconns_fast = false; break; case NO_LRU_CRAWLER: settings.lru_crawler = false; start_lru_crawler = false; break; case NO_LRU_MAINTAINER: start_lru_maintainer = false; settings.lru_segmented = false; break; #ifdef TLS case SSL_CERT: if (subopts_value == NULL) { fprintf(stderr, "Missing ssl_chain_cert argument\n"); return 1; } settings.ssl_chain_cert = strdup(subopts_value); break; case SSL_KEY: if (subopts_value == NULL) { fprintf(stderr, "Missing ssl_key argument\n"); return 1; } settings.ssl_key = strdup(subopts_value); break; case SSL_VERIFY_MODE: { if (subopts_value == NULL) { fprintf(stderr, "Missing ssl_verify_mode argument\n"); return 1; } int verify = 0; if (!safe_strtol(subopts_value, &verify)) { fprintf(stderr, "could not parse argument to ssl_verify_mode\n"); return 1; } switch(verify) { case 0: settings.ssl_verify_mode = SSL_VERIFY_NONE; break; case 1: settings.ssl_verify_mode = SSL_VERIFY_PEER; break; case 2: settings.ssl_verify_mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT; break; case 3: settings.ssl_verify_mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT | SSL_VERIFY_CLIENT_ONCE; break; default: fprintf(stderr, "Invalid ssl_verify_mode. Use help to see valid options.\n"); return 1; } break; } case SSL_KEYFORM: if (subopts_value == NULL) { fprintf(stderr, "Missing ssl_keyformat argument\n"); return 1; } if (!safe_strtol(subopts_value, &settings.ssl_keyformat)) { fprintf(stderr, "could not parse argument to ssl_keyformat\n"); return 1; } break; case SSL_CIPHERS: if (subopts_value == NULL) { fprintf(stderr, "Missing ssl_ciphers argument\n"); return 1; } settings.ssl_ciphers = strdup(subopts_value); break; case SSL_CA_CERT: if (subopts_value == NULL) { fprintf(stderr, "Missing ssl_ca_cert argument\n"); return 1; } settings.ssl_ca_cert = strdup(subopts_value); break; case SSL_WBUF_SIZE: if (subopts_value == NULL) { fprintf(stderr, "Missing ssl_wbuf_size argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.ssl_wbuf_size)) { fprintf(stderr, "could not parse argument to ssl_wbuf_size\n"); return 1; } settings.ssl_wbuf_size *= 1024; /* kilobytes */ break; #endif #ifdef EXTSTORE case EXT_PAGE_SIZE: if (storage_file) { fprintf(stderr, "Must specify ext_page_size before any ext_path arguments\n"); return 1; } if (subopts_value == NULL) { fprintf(stderr, "Missing ext_page_size argument\n"); return 1; } if (!safe_strtoul(subopts_value, &ext_cf.page_size)) { fprintf(stderr, "could not parse argument to ext_page_size\n"); return 1; } ext_cf.page_size *= 1024 * 1024; /* megabytes */ break; case EXT_WBUF_SIZE: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_wbuf_size argument\n"); return 1; } if (!safe_strtoul(subopts_value, &ext_cf.wbuf_size)) { fprintf(stderr, "could not parse argument to ext_wbuf_size\n"); return 1; } ext_cf.wbuf_size *= 1024 * 1024; /* megabytes */ settings.ext_wbuf_size = ext_cf.wbuf_size; break; case EXT_THREADS: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_threads argument\n"); return 1; } if (!safe_strtoul(subopts_value, &ext_cf.io_threadcount)) { fprintf(stderr, "could not parse argument to ext_threads\n"); return 1; } break; case EXT_IO_DEPTH: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_io_depth argument\n"); return 1; } if (!safe_strtoul(subopts_value, &ext_cf.io_depth)) { fprintf(stderr, "could not parse argument to ext_io_depth\n"); return 1; } break; case EXT_ITEM_SIZE: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_item_size argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.ext_item_size)) { fprintf(stderr, "could not parse argument to ext_item_size\n"); return 1; } break; case EXT_ITEM_AGE: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_item_age argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.ext_item_age)) { fprintf(stderr, "could not parse argument to ext_item_age\n"); return 1; } break; case EXT_LOW_TTL: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_low_ttl argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.ext_low_ttl)) { fprintf(stderr, "could not parse argument to ext_low_ttl\n"); return 1; } break; case EXT_RECACHE_RATE: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_recache_rate argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.ext_recache_rate)) { fprintf(stderr, "could not parse argument to ext_recache_rate\n"); return 1; } break; case EXT_COMPACT_UNDER: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_compact_under argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.ext_compact_under)) { fprintf(stderr, "could not parse argument to ext_compact_under\n"); return 1; } break; case EXT_DROP_UNDER: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_drop_under argument\n"); return 1; } if (!safe_strtoul(subopts_value, &settings.ext_drop_under)) { fprintf(stderr, "could not parse argument to ext_drop_under\n"); return 1; } break; case EXT_MAX_FRAG: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_max_frag argument\n"); return 1; } if (!safe_strtod(subopts_value, &settings.ext_max_frag)) { fprintf(stderr, "could not parse argument to ext_max_frag\n"); return 1; } break; case SLAB_AUTOMOVE_FREERATIO: if (subopts_value == NULL) { fprintf(stderr, "Missing slab_automove_freeratio argument\n"); return 1; } if (!safe_strtod(subopts_value, &settings.slab_automove_freeratio)) { fprintf(stderr, "could not parse argument to slab_automove_freeratio\n"); return 1; } break; case EXT_DROP_UNREAD: settings.ext_drop_unread = true; break; case EXT_PATH: if (subopts_value) { struct extstore_conf_file *tmp = storage_conf_parse(subopts_value, ext_cf.page_size); if (tmp == NULL) { fprintf(stderr, "failed to parse ext_path argument\n"); return 1; } if (storage_file != NULL) { tmp->next = storage_file; } storage_file = tmp; } else { fprintf(stderr, "missing argument to ext_path, ie: ext_path=/d/file:5G\n"); return 1; } break; #endif case MODERN: /* currently no new defaults */ break; case NO_MODERN: if (!slab_chunk_size_changed) { settings.slab_chunk_size_max = settings.slab_page_size; } settings.slab_reassign = false; settings.slab_automove = 0; settings.maxconns_fast = false; settings.lru_segmented = false; hash_type = JENKINS_HASH; start_lru_crawler = false; start_lru_maintainer = false; break; case NO_DROP_PRIVILEGES: settings.drop_privileges = false; break; case DROP_PRIVILEGES: settings.drop_privileges = true; break; #ifdef MEMCACHED_DEBUG case RELAXED_PRIVILEGES: settings.relaxed_privileges = true; break; #endif default: printf("Illegal suboption \"%s\"\n", subopts_value); return 1; } } free(subopts_orig); break; default: fprintf(stderr, "Illegal argument \"%c\"\n", c); return 1; } } if (settings.item_size_max < ITEM_SIZE_MAX_LOWER_LIMIT) { fprintf(stderr, "Item max size cannot be less than 1024 bytes.\n"); exit(EX_USAGE); } if (settings.item_size_max > (settings.maxbytes / 2)) { fprintf(stderr, "Cannot set item size limit higher than 1/2 of memory max.\n"); exit(EX_USAGE); } if (settings.item_size_max > (ITEM_SIZE_MAX_UPPER_LIMIT)) { fprintf(stderr, "Cannot set item size limit higher than a gigabyte.\n"); exit(EX_USAGE); } if (settings.item_size_max > 1024 * 1024) { if (!slab_chunk_size_changed) { // Ideal new default is 16k, but needs stitching. settings.slab_chunk_size_max = settings.slab_page_size / 2; } } if (settings.slab_chunk_size_max > settings.item_size_max) { fprintf(stderr, "slab_chunk_max (bytes: %d) cannot be larger than -I (item_size_max %d)\n", settings.slab_chunk_size_max, settings.item_size_max); exit(EX_USAGE); } if (settings.item_size_max % settings.slab_chunk_size_max != 0) { fprintf(stderr, "-I (item_size_max: %d) must be evenly divisible by slab_chunk_max (bytes: %d)\n", settings.item_size_max, settings.slab_chunk_size_max); exit(EX_USAGE); } if (settings.slab_page_size % settings.slab_chunk_size_max != 0) { fprintf(stderr, "slab_chunk_max (bytes: %d) must divide evenly into %d (slab_page_size)\n", settings.slab_chunk_size_max, settings.slab_page_size); exit(EX_USAGE); } #ifdef EXTSTORE if (storage_file) { if (settings.item_size_max > ext_cf.wbuf_size) { fprintf(stderr, "-I (item_size_max: %d) cannot be larger than ext_wbuf_size: %d\n", settings.item_size_max, ext_cf.wbuf_size); exit(EX_USAGE); } if (settings.udpport) { fprintf(stderr, "Cannot use UDP with extstore enabled (-U 0 to disable)\n"); exit(EX_USAGE); } } #endif // Reserve this for the new default. If factor size hasn't changed, use // new default. /*if (settings.slab_chunk_size_max == 16384 && settings.factor == 1.25) { settings.factor = 1.08; }*/ if (slab_sizes_unparsed != NULL) { // want the unedited string for restart code. char *temp = strdup(slab_sizes_unparsed); if (_parse_slab_sizes(slab_sizes_unparsed, slab_sizes)) { use_slab_sizes = true; if (meta->slab_config) { free(meta->slab_config); } meta->slab_config = temp; } else { exit(EX_USAGE); } } else if (!meta->slab_config) { // using the default factor. meta->slab_config = "1.25"; } if (settings.hot_lru_pct + settings.warm_lru_pct > 80) { fprintf(stderr, "hot_lru_pct + warm_lru_pct cannot be more than 80%% combined\n"); exit(EX_USAGE); } if (settings.temp_lru && !start_lru_maintainer) { fprintf(stderr, "temporary_ttl requires lru_maintainer to be enabled\n"); exit(EX_USAGE); } if (hash_init(hash_type) != 0) { fprintf(stderr, "Failed to initialize hash_algorithm!\n"); exit(EX_USAGE); } /* * Use one workerthread to serve each UDP port if the user specified * multiple ports */ if (settings.inter != NULL && strchr(settings.inter, ',')) { settings.num_threads_per_udp = 1; } else { settings.num_threads_per_udp = settings.num_threads; } if (settings.sasl) { if (!protocol_specified) { settings.binding_protocol = binary_prot; } else { if (settings.binding_protocol != binary_prot) { fprintf(stderr, "ERROR: You cannot allow the ASCII protocol while using SASL.\n"); exit(EX_USAGE); } } if (settings.udpport) { fprintf(stderr, "ERROR: Cannot enable UDP while using binary SASL authentication.\n"); exit(EX_USAGE); } } if (settings.auth_file) { if (!protocol_specified) { settings.binding_protocol = ascii_prot; } else { if (settings.binding_protocol != ascii_prot) { fprintf(stderr, "ERROR: You cannot allow the BINARY protocol while using ascii authentication tokens.\n"); exit(EX_USAGE); } } } if (udp_specified && settings.udpport != 0 && !tcp_specified) { settings.port = settings.udpport; } #ifdef TLS /* * Setup SSL if enabled */ if (settings.ssl_enabled) { if (!settings.port) { fprintf(stderr, "ERROR: You cannot enable SSL without a TCP port.\n"); exit(EX_USAGE); } // openssl init methods. SSL_load_error_strings(); SSLeay_add_ssl_algorithms(); // Initiate the SSL context. ssl_init(); } #endif if (maxcore != 0) { struct rlimit rlim_new; /* * First try raising to infinity; if that fails, try bringing * the soft limit to the hard. */ if (getrlimit(RLIMIT_CORE, &rlim) == 0) { rlim_new.rlim_cur = rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new)!= 0) { /* failed. try raising just to the old max */ rlim_new.rlim_cur = rlim_new.rlim_max = rlim.rlim_max; (void)setrlimit(RLIMIT_CORE, &rlim_new); } } /* * getrlimit again to see what we ended up with. Only fail if * the soft limit ends up 0, because then no core files will be * created at all. */ if ((getrlimit(RLIMIT_CORE, &rlim) != 0) || rlim.rlim_cur == 0) { fprintf(stderr, "failed to ensure corefile creation\n"); exit(EX_OSERR); } } /* * If needed, increase rlimits to allow as many connections * as needed. */ if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { fprintf(stderr, "failed to getrlimit number of files\n"); exit(EX_OSERR); } else { rlim.rlim_cur = settings.maxconns; rlim.rlim_max = settings.maxconns; if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) { fprintf(stderr, "failed to set rlimit for open files. Try starting as root or requesting smaller maxconns value.\n"); exit(EX_OSERR); } } /* lose root privileges if we have them */ if (getuid() == 0 || geteuid() == 0) { if (username == 0 || *username == '\0') { fprintf(stderr, "can't run as root without the -u switch\n"); exit(EX_USAGE); } if ((pw = getpwnam(username)) == 0) { fprintf(stderr, "can't find the user %s to switch to\n", username); exit(EX_NOUSER); } if (setgroups(0, NULL) < 0) { /* setgroups may fail with EPERM, indicating we are already in a * minimally-privileged state. In that case we continue. For all * other failure codes we exit. * * Note that errno is stored here because fprintf may change it. */ bool should_exit = errno != EPERM; fprintf(stderr, "failed to drop supplementary groups: %s\n", strerror(errno)); if (should_exit) { exit(EX_OSERR); } } if (setgid(pw->pw_gid) < 0 || setuid(pw->pw_uid) < 0) { fprintf(stderr, "failed to assume identity of user %s\n", username); exit(EX_OSERR); } } /* Initialize Sasl if -S was specified */ if (settings.sasl) { init_sasl(); } /* daemonize if requested */ /* if we want to ensure our ability to dump core, don't chdir to / */ if (do_daemonize) { if (sigignore(SIGHUP) == -1) { perror("Failed to ignore SIGHUP"); } if (daemonize(maxcore, settings.verbose) == -1) { fprintf(stderr, "failed to daemon() in order to daemonize\n"); exit(EXIT_FAILURE); } } /* lock paged memory if needed */ if (lock_memory) { #ifdef HAVE_MLOCKALL int res = mlockall(MCL_CURRENT | MCL_FUTURE); if (res != 0) { fprintf(stderr, "warning: -k invalid, mlockall() failed: %s\n", strerror(errno)); } #else fprintf(stderr, "warning: -k invalid, mlockall() not supported on this platform. proceeding without.\n"); #endif } /* initialize main thread libevent instance */ #if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= 0x02000101 /* If libevent version is larger/equal to 2.0.2-alpha, use newer version */ struct event_config *ev_config; ev_config = event_config_new(); event_config_set_flag(ev_config, EVENT_BASE_FLAG_NOLOCK); main_base = event_base_new_with_config(ev_config); event_config_free(ev_config); #else /* Otherwise, use older API */ main_base = event_init(); #endif /* Load initial auth file if required */ if (settings.auth_file) { if (settings.udpport) { fprintf(stderr, "Cannot use UDP with ascii authentication enabled (-U 0 to disable)\n"); exit(EX_USAGE); } switch (authfile_load(settings.auth_file)) { case AUTHFILE_MISSING: // fall through. case AUTHFILE_OPENFAIL: vperror("Could not open authfile [%s] for reading", settings.auth_file); exit(EXIT_FAILURE); break; case AUTHFILE_OOM: fprintf(stderr, "Out of memory reading password file: %s", settings.auth_file); exit(EXIT_FAILURE); break; case AUTHFILE_MALFORMED: fprintf(stderr, "Authfile [%s] has a malformed entry. Should be 'user:password'", settings.auth_file); exit(EXIT_FAILURE); break; case AUTHFILE_OK: break; } } /* initialize other stuff */ stats_init(); logger_init(); conn_init(); bool reuse_mem = false; void *mem_base = NULL; bool prefill = false; if (memory_file != NULL) { preallocate = true; // Easier to manage memory if we prefill the global pool when reusing. prefill = true; restart_register("main", _mc_meta_load_cb, _mc_meta_save_cb, meta); reuse_mem = restart_mmap_open(settings.maxbytes, memory_file, &mem_base); // The "save" callback gets called when we're closing out the mmap, // but we don't know what the mmap_base is until after we call open. // So we pass the struct above but have to fill it in here so the // data's available during the save routine. meta->mmap_base = mem_base; // Also, the callbacks for load() run before _open returns, so we // should have the old base in 'meta' as of here. } // Initialize the hash table _after_ checking restart metadata. // We override the hash table start argument with what was live // previously, to avoid filling a huge set of items into a tiny hash // table. assoc_init(settings.hashpower_init); #ifdef EXTSTORE if (storage_file && reuse_mem) { fprintf(stderr, "[restart] memory restart with extstore not presently supported.\n"); reuse_mem = false; } #endif slabs_init(settings.maxbytes, settings.factor, preallocate, use_slab_sizes ? slab_sizes : NULL, mem_base, reuse_mem); #ifdef EXTSTORE if (storage_file) { enum extstore_res eres; if (settings.ext_compact_under == 0) { // If changing the default fraction (4), change the help text as well. settings.ext_compact_under = storage_file->page_count / 4; /* Only rescues non-COLD items if below this threshold */ settings.ext_drop_under = storage_file->page_count / 4; } crc32c_init(); /* Init free chunks to zero. */ for (int x = 0; x < MAX_NUMBER_OF_SLAB_CLASSES; x++) { settings.ext_free_memchunks[x] = 0; } storage = extstore_init(storage_file, &ext_cf, &eres); if (storage == NULL) { fprintf(stderr, "Failed to initialize external storage: %s\n", extstore_err(eres)); if (eres == EXTSTORE_INIT_OPEN_FAIL) { perror("extstore open"); } exit(EXIT_FAILURE); } ext_storage = storage; /* page mover algorithm for extstore needs memory prefilled */ prefill = true; } #endif if (settings.drop_privileges) { setup_privilege_violations_handler(); } if (prefill) slabs_prefill_global(); /* In restartable mode and we've decided to issue a fixup on memory */ if (memory_file != NULL && reuse_mem) { mc_ptr_t old_base = meta->old_base; assert(old_base == meta->old_base); // should've pulled in process_started from meta file. process_started = meta->process_started; // TODO: must be a more canonical way of serializing/deserializing // pointers? passing through uint64_t should work, and we're not // annotating the pointer with anything, but it's still slightly // insane. restart_fixup((void *)old_base); } /* * ignore SIGPIPE signals; we can use errno == EPIPE if we * need that information */ if (sigignore(SIGPIPE) == -1) { perror("failed to ignore SIGPIPE; sigaction"); exit(EX_OSERR); } /* start up worker threads if MT mode */ #ifdef EXTSTORE slabs_set_storage(storage); memcached_thread_init(settings.num_threads, storage); init_lru_crawler(storage); #else memcached_thread_init(settings.num_threads, NULL); init_lru_crawler(NULL); #endif if (start_assoc_maint && start_assoc_maintenance_thread() == -1) { exit(EXIT_FAILURE); } if (start_lru_crawler && start_item_crawler_thread() != 0) { fprintf(stderr, "Failed to enable LRU crawler thread\n"); exit(EXIT_FAILURE); } #ifdef EXTSTORE if (storage && start_storage_compact_thread(storage) != 0) { fprintf(stderr, "Failed to start storage compaction thread\n"); exit(EXIT_FAILURE); } if (storage && start_storage_write_thread(storage) != 0) { fprintf(stderr, "Failed to start storage writer thread\n"); exit(EXIT_FAILURE); } if (start_lru_maintainer && start_lru_maintainer_thread(storage) != 0) { #else if (start_lru_maintainer && start_lru_maintainer_thread(NULL) != 0) { #endif fprintf(stderr, "Failed to enable LRU maintainer thread\n"); free(meta); return 1; } if (settings.slab_reassign && start_slab_maintenance_thread() == -1) { exit(EXIT_FAILURE); } if (settings.idle_timeout && start_conn_timeout_thread() == -1) { exit(EXIT_FAILURE); } /* initialise clock event */ #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) { struct timespec ts; if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) { monotonic = true; monotonic_start = ts.tv_sec; // Monotonic clock needs special handling for restarts. // We get a start time at an arbitrary place, so we need to // restore the original time delta, which is always "now" - _start if (reuse_mem) { // the running timespan at stop time + the time we think we // were stopped. monotonic_start -= meta->current_time + meta->time_delta; } else { monotonic_start -= ITEM_UPDATE_INTERVAL + 2; } } } #endif clock_handler(0, 0, 0); /* create unix mode sockets after dropping privileges */ if (settings.socketpath != NULL) { errno = 0; if (server_socket_unix(settings.socketpath,settings.access)) { vperror("failed to listen on UNIX socket: %s", settings.socketpath); exit(EX_OSERR); } } /* create the listening socket, bind it, and init */ if (settings.socketpath == NULL) { const char *portnumber_filename = getenv("MEMCACHED_PORT_FILENAME"); char *temp_portnumber_filename = NULL; size_t len; FILE *portnumber_file = NULL; if (portnumber_filename != NULL) { len = strlen(portnumber_filename)+4+1; temp_portnumber_filename = malloc(len); snprintf(temp_portnumber_filename, len, "%s.lck", portnumber_filename); portnumber_file = fopen(temp_portnumber_filename, "a"); if (portnumber_file == NULL) { fprintf(stderr, "Failed to open \"%s\": %s\n", temp_portnumber_filename, strerror(errno)); } } errno = 0; if (settings.port && server_sockets(settings.port, tcp_transport, portnumber_file)) { vperror("failed to listen on TCP port %d", settings.port); exit(EX_OSERR); } /* * initialization order: first create the listening sockets * (may need root on low ports), then drop root if needed, * then daemonize if needed, then init libevent (in some cases * descriptors created by libevent wouldn't survive forking). */ /* create the UDP listening socket and bind it */ errno = 0; if (settings.udpport && server_sockets(settings.udpport, udp_transport, portnumber_file)) { vperror("failed to listen on UDP port %d", settings.udpport); exit(EX_OSERR); } if (portnumber_file) { fclose(portnumber_file); rename(temp_portnumber_filename, portnumber_filename); } if (temp_portnumber_filename) free(temp_portnumber_filename); } /* Give the sockets a moment to open. I know this is dumb, but the error * is only an advisory. */ usleep(1000); if (stats_state.curr_conns + stats_state.reserved_fds >= settings.maxconns - 1) { fprintf(stderr, "Maxconns setting is too low, use -c to increase.\n"); exit(EXIT_FAILURE); } if (pid_file != NULL) { save_pid(pid_file); } /* Drop privileges no longer needed */ if (settings.drop_privileges) { drop_privileges(); } /* Initialize the uriencode lookup table. */ uriencode_init(); /* enter the event loop */ while (!stop_main_loop) { if (event_base_loop(main_base, EVLOOP_ONCE) != 0) { retval = EXIT_FAILURE; break; } } fprintf(stderr, "Gracefully stopping\n"); stop_threads(); int i; // FIXME: make a function callable from threads.c for (i = 0; i < max_fds; i++) { if (conns[i] && conns[i]->state != conn_closed) { conn_close(conns[i]); } } if (memory_file != NULL) { restart_mmap_close(); } /* remove the PID file if we're a daemon */ if (do_daemonize) remove_pidfile(pid_file); /* Clean up strdup() call for bind() address */ if (settings.inter) free(settings.inter); /* cleanup base */ event_base_free(main_base); free(meta); return retval; } memcached-1.5.22/crc32c.c0000664000175000017500000003237613553270304011672 00000000000000/* crc32c.c -- compute CRC-32C using the Intel crc32 instruction * Copyright (C) 2013 Mark Adler * Version 1.1 1 Aug 2013 Mark Adler */ /* This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. Mark Adler madler@alumni.caltech.edu */ /* Use hardware CRC instruction on Intel SSE 4.2 processors. This computes a CRC-32C, *not* the CRC-32 used by Ethernet and zip, gzip, etc. A software version is provided as a fall-back, as well as for speed comparisons. */ /* Version history: 1.0 10 Feb 2013 First version 1.1 1 Aug 2013 Correct comments on why three crc instructions in parallel */ /* This version has been modified by dormando for inclusion in memcached */ #include #include #include #include #include #include "config.h" #if defined(__linux__) && defined(__aarch64__) #include #endif #include "crc32c.h" /* CRC-32C (iSCSI) polynomial in reversed bit order. */ #define POLY 0x82f63b78 /* Table for a quadword-at-a-time software crc. */ static pthread_once_t crc32c_once_sw = PTHREAD_ONCE_INIT; static uint32_t crc32c_table[8][256]; /* Construct table for software CRC-32C calculation. */ static void crc32c_init_sw(void) { uint32_t n, crc, k; for (n = 0; n < 256; n++) { crc = n; crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; crc32c_table[0][n] = crc; } for (n = 0; n < 256; n++) { crc = crc32c_table[0][n]; for (k = 1; k < 8; k++) { crc = crc32c_table[0][crc & 0xff] ^ (crc >> 8); crc32c_table[k][n] = crc; } } } /* Table-driven software version as a fall-back. This is about 15 times slower than using the hardware instructions. This assumes little-endian integers, as is the case on Intel processors that the assembler code here is for. */ static uint32_t crc32c_sw(uint32_t crci, const void *buf, size_t len) { const unsigned char *next = buf; uint64_t crc; pthread_once(&crc32c_once_sw, crc32c_init_sw); crc = crci ^ 0xffffffff; while (len && ((uintptr_t)next & 7) != 0) { crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); len--; } while (len >= 8) { crc ^= *(uint64_t *)next; crc = crc32c_table[7][crc & 0xff] ^ crc32c_table[6][(crc >> 8) & 0xff] ^ crc32c_table[5][(crc >> 16) & 0xff] ^ crc32c_table[4][(crc >> 24) & 0xff] ^ crc32c_table[3][(crc >> 32) & 0xff] ^ crc32c_table[2][(crc >> 40) & 0xff] ^ crc32c_table[1][(crc >> 48) & 0xff] ^ crc32c_table[0][crc >> 56]; next += 8; len -= 8; } while (len) { crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); len--; } return (uint32_t)crc ^ 0xffffffff; } /* Hardware CRC support for aarch64 platform */ #if defined(__linux__) && defined(__aarch64__) && defined(ARM_CRC32) #define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(+value)) #define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(+value)) #define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(+value)) #define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(+value)) #ifndef HWCAP_CRC32 #define HWCAP_CRC32 (1 << 7) #endif /* HWCAP for crc32 */ static uint32_t crc32c_hw_aarch64(uint32_t crc, const void* buf, size_t len) { const uint8_t* p_buf = buf; uint64_t crc64bit = crc; for (size_t i = 0; i < len / sizeof(uint64_t); i++) { CRC32CX(crc64bit, *(uint64_t*) p_buf); p_buf += sizeof(uint64_t); } uint32_t crc32bit = (uint32_t) crc64bit; len &= sizeof(uint64_t) - 1; switch (len) { case 7: CRC32CB(crc32bit, *p_buf++); case 6: CRC32CH(crc32bit, *(uint16_t*) p_buf); p_buf += 2; case 4: CRC32CW(crc32bit, *(uint32_t*) p_buf); break; case 3: CRC32CB(crc32bit, *p_buf++); case 2: CRC32CH(crc32bit, *(uint16_t*) p_buf); break; case 5: CRC32CW(crc32bit, *(uint32_t*) p_buf); p_buf += 4; case 1: CRC32CB(crc32bit, *p_buf); break; case 0: break; } return crc32bit; } #endif /* Apply if the platform is intel */ #if defined(__X86_64__)||defined(__x86_64__)||defined(__ia64__) /* Multiply a matrix times a vector over the Galois field of two elements, GF(2). Each element is a bit in an unsigned integer. mat must have at least as many entries as the power of two for most significant one bit in vec. */ static inline uint32_t gf2_matrix_times(uint32_t *mat, uint32_t vec) { uint32_t sum; sum = 0; while (vec) { if (vec & 1) sum ^= *mat; vec >>= 1; mat++; } return sum; } /* Multiply a matrix by itself over GF(2). Both mat and square must have 32 rows. */ static inline void gf2_matrix_square(uint32_t *square, uint32_t *mat) { int n; for (n = 0; n < 32; n++) square[n] = gf2_matrix_times(mat, mat[n]); } /* Construct an operator to apply len zeros to a crc. len must be a power of two. If len is not a power of two, then the result is the same as for the largest power of two less than len. The result for len == 0 is the same as for len == 1. A version of this routine could be easily written for any len, but that is not needed for this application. */ static void crc32c_zeros_op(uint32_t *even, size_t len) { int n; uint32_t row; uint32_t odd[32]; /* odd-power-of-two zeros operator */ /* put operator for one zero bit in odd */ odd[0] = POLY; /* CRC-32C polynomial */ row = 1; for (n = 1; n < 32; n++) { odd[n] = row; row <<= 1; } /* put operator for two zero bits in even */ gf2_matrix_square(even, odd); /* put operator for four zero bits in odd */ gf2_matrix_square(odd, even); /* first square will put the operator for one zero byte (eight zero bits), in even -- next square puts operator for two zero bytes in odd, and so on, until len has been rotated down to zero */ do { gf2_matrix_square(even, odd); len >>= 1; if (len == 0) return; gf2_matrix_square(odd, even); len >>= 1; } while (len); /* answer ended up in odd -- copy to even */ for (n = 0; n < 32; n++) even[n] = odd[n]; } /* Take a length and build four lookup tables for applying the zeros operator for that length, byte-by-byte on the operand. */ static void crc32c_zeros(uint32_t zeros[][256], size_t len) { uint32_t n; uint32_t op[32]; crc32c_zeros_op(op, len); for (n = 0; n < 256; n++) { zeros[0][n] = gf2_matrix_times(op, n); zeros[1][n] = gf2_matrix_times(op, n << 8); zeros[2][n] = gf2_matrix_times(op, n << 16); zeros[3][n] = gf2_matrix_times(op, n << 24); } } /* Apply the zeros operator table to crc. */ static inline uint32_t crc32c_shift(uint32_t zeros[][256], uint32_t crc) { return zeros[0][crc & 0xff] ^ zeros[1][(crc >> 8) & 0xff] ^ zeros[2][(crc >> 16) & 0xff] ^ zeros[3][crc >> 24]; } /* Block sizes for three-way parallel crc computation. LONG and SHORT must both be powers of two. The associated string constants must be set accordingly, for use in constructing the assembler instructions. */ #define LONG 8192 #define LONGx1 "8192" #define LONGx2 "16384" #define SHORT 256 #define SHORTx1 "256" #define SHORTx2 "512" /* Tables for hardware crc that shift a crc by LONG and SHORT zeros. */ static pthread_once_t crc32c_once_hw = PTHREAD_ONCE_INIT; static uint32_t crc32c_long[4][256]; static uint32_t crc32c_short[4][256]; /* Initialize tables for shifting crcs. */ static void crc32c_init_hw(void) { crc32c_zeros(crc32c_long, LONG); crc32c_zeros(crc32c_short, SHORT); } /* Compute CRC-32C using the Intel hardware instruction. */ static uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len) { const unsigned char *next = buf; const unsigned char *end; uint64_t crc0, crc1, crc2; /* need to be 64 bits for crc32q */ /* populate shift tables the first time through */ pthread_once(&crc32c_once_hw, crc32c_init_hw); /* pre-process the crc */ crc0 = crc ^ 0xffffffff; /* compute the crc for up to seven leading bytes to bring the data pointer to an eight-byte boundary */ while (len && ((uintptr_t)next & 7) != 0) { __asm__("crc32b\t" "(%1), %0" : "=r"(crc0) : "r"(next), "0"(crc0)); next++; len--; } /* compute the crc on sets of LONG*3 bytes, executing three independent crc instructions, each on LONG bytes -- this is optimized for the Nehalem, Westmere, Sandy Bridge, and Ivy Bridge architectures, which have a throughput of one crc per cycle, but a latency of three cycles */ while (len >= LONG*3) { crc1 = 0; crc2 = 0; end = next + LONG; do { __asm__("crc32q\t" "(%3), %0\n\t" "crc32q\t" LONGx1 "(%3), %1\n\t" "crc32q\t" LONGx2 "(%3), %2" : "=r"(crc0), "=r"(crc1), "=r"(crc2) : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2)); next += 8; } while (next < end); crc0 = crc32c_shift(crc32c_long, crc0) ^ crc1; crc0 = crc32c_shift(crc32c_long, crc0) ^ crc2; next += LONG*2; len -= LONG*3; } /* do the same thing, but now on SHORT*3 blocks for the remaining data less than a LONG*3 block */ while (len >= SHORT*3) { crc1 = 0; crc2 = 0; end = next + SHORT; do { __asm__("crc32q\t" "(%3), %0\n\t" "crc32q\t" SHORTx1 "(%3), %1\n\t" "crc32q\t" SHORTx2 "(%3), %2" : "=r"(crc0), "=r"(crc1), "=r"(crc2) : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2)); next += 8; } while (next < end); crc0 = crc32c_shift(crc32c_short, crc0) ^ crc1; crc0 = crc32c_shift(crc32c_short, crc0) ^ crc2; next += SHORT*2; len -= SHORT*3; } /* compute the crc on the remaining eight-byte units less than a SHORT*3 block */ end = next + (len - (len & 7)); while (next < end) { __asm__("crc32q\t" "(%1), %0" : "=r"(crc0) : "r"(next), "0"(crc0)); next += 8; } len &= 7; /* compute the crc for up to seven trailing bytes */ while (len) { __asm__("crc32b\t" "(%1), %0" : "=r"(crc0) : "r"(next), "0"(crc0)); next++; len--; } /* return a post-processed crc */ return (uint32_t)crc0 ^ 0xffffffff; } /* Check for SSE 4.2. SSE 4.2 was first supported in Nehalem processors introduced in November, 2008. This does not check for the existence of the cpuid instruction itself, which was introduced on the 486SL in 1992, so this will fail on earlier x86 processors. cpuid works on all Pentium and later processors. */ #define SSE42(have) \ do { \ uint32_t eax, ecx; \ eax = 1; \ __asm__("cpuid" \ : "=c"(ecx) \ : "a"(eax) \ : "%ebx", "%edx"); \ (have) = (ecx >> 20) & 1; \ } while (0) #endif /* Compute a CRC-32C. If the crc32 instruction is available, use the hardware version. Otherwise, use the software version. */ void crc32c_init(void) { #if defined(__X86_64__)||defined(__x86_64__)||defined(__ia64__) int sse42; SSE42(sse42); if (sse42) { crc32c = crc32c_hw; } else #endif /* Check if CRC instructions supported by aarch64 */ #if defined(__linux__) && defined(__aarch64__) && defined(ARM_CRC32) unsigned long hwcap = getauxval(AT_HWCAP); if (hwcap & HWCAP_CRC32) { crc32c = crc32c_hw_aarch64; } else #endif { crc32c = crc32c_sw; } } memcached-1.5.22/authfile.h0000664000175000017500000000052313575022500012403 00000000000000#ifndef AUTHFILE_H #define AUTHFILE_H enum authfile_ret { AUTHFILE_OK = 0, AUTHFILE_MISSING, AUTHFILE_OOM, AUTHFILE_OPENFAIL, AUTHFILE_MALFORMED, }; // FIXME: mc_authfile or something? enum authfile_ret authfile_load(const char *file); int authfile_check(const char *user, const char *pass); #endif /* AUTHFILE_H */ memcached-1.5.22/murmur3_hash.h0000664000175000017500000000125113461170555013226 00000000000000//----------------------------------------------------------------------------- // MurmurHash3 was written by Austin Appleby, and is placed in the public // domain. The author hereby disclaims copyright to this source code. #ifndef MURMURHASH3_H #define MURMURHASH3_H //----------------------------------------------------------------------------- // Platform-specific functions and macros #include #include //----------------------------------------------------------------------------- uint32_t MurmurHash3_x86_32(const void *key, size_t length); //----------------------------------------------------------------------------- #endif // MURMURHASH3_H memcached-1.5.22/daemon.c0000664000175000017500000000601213461170555012047 00000000000000/* $Header: /cvsroot/wikipedia/willow/src/bin/willow/daemon.c,v 1.1 2005/05/02 19:15:21 kateturner Exp $ */ /* $NetBSD: daemon.c,v 1.9 2003/08/07 16:42:46 agc Exp $ */ /*- * Copyright (c) 1990, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if defined __SUNPRO_C || defined __DECC || defined __HP_cc # pragma ident "@(#)$Header: /cvsroot/wikipedia/willow/src/bin/willow/daemon.c,v 1.1 2005/05/02 19:15:21 kateturner Exp $" # pragma ident "$NetBSD: daemon.c,v 1.9 2003/08/07 16:42:46 agc Exp $" #endif #include #include #include #include #include "memcached.h" int daemonize(int nochdir, int noclose) { int fd; switch (fork()) { case -1: return (-1); case 0: break; default: _exit(EXIT_SUCCESS); } if (setsid() == -1) return (-1); if (nochdir == 0) { if(chdir("/") != 0) { perror("chdir"); return (-1); } } if (noclose == 0 && (fd = open("/dev/null", O_RDWR, 0)) != -1) { if(dup2(fd, STDIN_FILENO) < 0) { perror("dup2 stdin"); return (-1); } if(dup2(fd, STDOUT_FILENO) < 0) { perror("dup2 stdout"); return (-1); } if(dup2(fd, STDERR_FILENO) < 0) { perror("dup2 stderr"); return (-1); } if (fd > STDERR_FILENO) { if(close(fd) < 0) { perror("close"); return (-1); } } } return (0); } memcached-1.5.22/logger.c0000664000175000017500000006700313575022500012062 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #include #include #include #include #include #include #include #if defined(__sun) #include #endif #include "memcached.h" #include "bipbuffer.h" #ifdef LOGGER_DEBUG #define L_DEBUG(...) \ do { \ fprintf(stderr, __VA_ARGS__); \ } while (0) #else #define L_DEBUG(...) #endif /* TODO: put this in a struct and ditch the global vars. */ static logger *logger_stack_head = NULL; static logger *logger_stack_tail = NULL; static unsigned int logger_count = 0; static volatile int do_run_logger_thread = 1; static pthread_t logger_tid; pthread_mutex_t logger_stack_lock = PTHREAD_MUTEX_INITIALIZER; pthread_key_t logger_key; #if !defined(HAVE_GCC_64ATOMICS) && !defined(__sun) pthread_mutex_t logger_atomics_mutex = PTHREAD_MUTEX_INITIALIZER; #endif #define WATCHER_LIMIT 20 logger_watcher *watchers[20]; struct pollfd watchers_pollfds[20]; int watcher_count = 0; /* Should this go somewhere else? */ static const entry_details default_entries[] = { [LOGGER_ASCII_CMD] = {LOGGER_TEXT_ENTRY, 512, LOG_RAWCMDS, "<%d %s"}, [LOGGER_EVICTION] = {LOGGER_EVICTION_ENTRY, 512, LOG_EVICTIONS, NULL}, [LOGGER_ITEM_GET] = {LOGGER_ITEM_GET_ENTRY, 512, LOG_FETCHERS, NULL}, [LOGGER_ITEM_STORE] = {LOGGER_ITEM_STORE_ENTRY, 512, LOG_MUTATIONS, NULL}, [LOGGER_CRAWLER_STATUS] = {LOGGER_TEXT_ENTRY, 512, LOG_SYSEVENTS, "type=lru_crawler crawler=%d lru=%s low_mark=%llu next_reclaims=%llu since_run=%u next_run=%d elapsed=%u examined=%llu reclaimed=%llu" }, [LOGGER_SLAB_MOVE] = {LOGGER_TEXT_ENTRY, 512, LOG_SYSEVENTS, "type=slab_move src=%d dst=%d" }, #ifdef EXTSTORE [LOGGER_EXTSTORE_WRITE] = {LOGGER_EXT_WRITE_ENTRY, 512, LOG_EVICTIONS, NULL}, [LOGGER_COMPACT_START] = {LOGGER_TEXT_ENTRY, 512, LOG_SYSEVENTS, "type=compact_start id=%lu version=%llu" }, [LOGGER_COMPACT_ABORT] = {LOGGER_TEXT_ENTRY, 512, LOG_SYSEVENTS, "type=compact_abort id=%lu" }, [LOGGER_COMPACT_READ_START] = {LOGGER_TEXT_ENTRY, 512, LOG_SYSEVENTS, "type=compact_read_start id=%lu offset=%llu" }, [LOGGER_COMPACT_READ_END] = {LOGGER_TEXT_ENTRY, 512, LOG_SYSEVENTS, "type=compact_read_end id=%lu offset=%llu rescues=%lu lost=%lu skipped=%lu" }, [LOGGER_COMPACT_END] = {LOGGER_TEXT_ENTRY, 512, LOG_SYSEVENTS, "type=compact_end id=%lu" }, [LOGGER_COMPACT_FRAGINFO] = {LOGGER_TEXT_ENTRY, 512, LOG_SYSEVENTS, "type=compact_fraginfo ratio=%.2f bytes=%lu" }, #endif }; #define WATCHER_ALL -1 static int logger_thread_poll_watchers(int force_poll, int watcher); /************************* * Util functions shared between bg thread and workers *************************/ /* Logger GID's can be used by watchers to put logs back into strict order */ static uint64_t logger_gid = 0; uint64_t logger_get_gid(void) { #ifdef HAVE_GCC_64ATOMICS return __sync_add_and_fetch(&logger_gid, 1); #elif defined(__sun) return atomic_inc_64_nv(&logger_gid); #else mutex_lock(&logger_atomics_mutex); uint64_t res = ++logger_gid; mutex_unlock(&logger_atomics_mutex); return res; #endif } void logger_set_gid(uint64_t gid) { #ifdef HAVE_GCC_64ATOMICS __sync_add_and_fetch(&logger_gid, gid); #elif defined(__sun) atomic_add_64(&logger_gid); #else mutex_lock(&logger_atomics_mutex); logger_gid = gid; mutex_unlock(&logger_atomics_mutex); #endif } /* TODO: genericize lists. would be nice to import queue.h if the impact is * studied... otherwise can just write a local one. */ /* Add to the list of threads with a logger object */ static void logger_link_q(logger *l) { pthread_mutex_lock(&logger_stack_lock); assert(l != logger_stack_head); l->prev = 0; l->next = logger_stack_head; if (l->next) l->next->prev = l; logger_stack_head = l; if (logger_stack_tail == 0) logger_stack_tail = l; logger_count++; pthread_mutex_unlock(&logger_stack_lock); return; } /* Remove from the list of threads with a logger object */ /*static void logger_unlink_q(logger *l) { pthread_mutex_lock(&logger_stack_lock); if (logger_stack_head == l) { assert(l->prev == 0); logger_stack_head = l->next; } if (logger_stack_tail == l) { assert(l->next == 0); logger_stack_tail = l->prev; } assert(l->next != l); assert(l->prev != l); if (l->next) l->next->prev = l->prev; if (l->prev) l->prev->next = l->next; logger_count--; pthread_mutex_unlock(&logger_stack_lock); return; }*/ /* Called with logger stack locked. * Iterates over every watcher collecting enabled flags. */ static void logger_set_flags(void) { logger *l = NULL; int x = 0; uint16_t f = 0; /* logger eflags */ for (x = 0; x < WATCHER_LIMIT; x++) { logger_watcher *w = watchers[x]; if (w == NULL) continue; f |= w->eflags; } for (l = logger_stack_head; l != NULL; l=l->next) { pthread_mutex_lock(&l->mutex); l->eflags = f; pthread_mutex_unlock(&l->mutex); } return; } /************************* * Logger background thread functions. Aggregates per-worker buffers and * writes to any watchers. *************************/ #define LOGGER_PARSE_SCRATCH 4096 static int _logger_thread_parse_ise(logentry *e, char *scratch) { int total; const char *cmd = "na"; char keybuf[KEY_MAX_URI_ENCODED_LENGTH]; struct logentry_item_store *le = (struct logentry_item_store *) e->data; const char * const status_map[] = { "not_stored", "stored", "exists", "not_found", "too_large", "no_memory" }; const char * const cmd_map[] = { "null", "add", "set", "replace", "append", "prepend", "cas" }; if (le->cmd <= 6) cmd = cmd_map[le->cmd]; uriencode(le->key, keybuf, le->nkey, KEY_MAX_URI_ENCODED_LENGTH); total = snprintf(scratch, LOGGER_PARSE_SCRATCH, "ts=%d.%d gid=%llu type=item_store key=%s status=%s cmd=%s ttl=%u clsid=%u cfd=%d\n", (int)e->tv.tv_sec, (int)e->tv.tv_usec, (unsigned long long) e->gid, keybuf, status_map[le->status], cmd, le->ttl, le->clsid, le->sfd); return total; } static int _logger_thread_parse_ige(logentry *e, char *scratch) { int total; struct logentry_item_get *le = (struct logentry_item_get *) e->data; char keybuf[KEY_MAX_URI_ENCODED_LENGTH]; const char * const was_found_map[] = { "not_found", "found", "flushed", "expired" }; uriencode(le->key, keybuf, le->nkey, KEY_MAX_URI_ENCODED_LENGTH); total = snprintf(scratch, LOGGER_PARSE_SCRATCH, "ts=%d.%d gid=%llu type=item_get key=%s status=%s clsid=%u cfd=%d\n", (int)e->tv.tv_sec, (int)e->tv.tv_usec, (unsigned long long) e->gid, keybuf, was_found_map[le->was_found], le->clsid, le->sfd); return total; } static int _logger_thread_parse_ee(logentry *e, char *scratch) { int total; char keybuf[KEY_MAX_URI_ENCODED_LENGTH]; struct logentry_eviction *le = (struct logentry_eviction *) e->data; uriencode(le->key, keybuf, le->nkey, KEY_MAX_URI_ENCODED_LENGTH); total = snprintf(scratch, LOGGER_PARSE_SCRATCH, "ts=%d.%d gid=%llu type=eviction key=%s fetch=%s ttl=%lld la=%d clsid=%u\n", (int)e->tv.tv_sec, (int)e->tv.tv_usec, (unsigned long long) e->gid, keybuf, (le->it_flags & ITEM_FETCHED) ? "yes" : "no", (long long int)le->exptime, le->latime, le->clsid); return total; } #ifdef EXTSTORE static int _logger_thread_parse_extw(logentry *e, char *scratch) { int total; char keybuf[KEY_MAX_URI_ENCODED_LENGTH]; struct logentry_ext_write *le = (struct logentry_ext_write *) e->data; uriencode(le->key, keybuf, le->nkey, KEY_MAX_URI_ENCODED_LENGTH); total = snprintf(scratch, LOGGER_PARSE_SCRATCH, "ts=%d.%d gid=%llu type=extwrite key=%s fetch=%s ttl=%lld la=%d clsid=%u bucket=%u\n", (int)e->tv.tv_sec, (int)e->tv.tv_usec, (unsigned long long) e->gid, keybuf, (le->it_flags & ITEM_FETCHED) ? "yes" : "no", (long long int)le->exptime, le->latime, le->clsid, le->bucket); return total; } #endif /* Completes rendering of log line. */ static enum logger_parse_entry_ret logger_thread_parse_entry(logentry *e, struct logger_stats *ls, char *scratch, int *scratch_len) { int total = 0; switch (e->event) { case LOGGER_TEXT_ENTRY: total = snprintf(scratch, LOGGER_PARSE_SCRATCH, "ts=%d.%d gid=%llu %s\n", (int)e->tv.tv_sec, (int)e->tv.tv_usec, (unsigned long long) e->gid, (char *) e->data); break; case LOGGER_EVICTION_ENTRY: total = _logger_thread_parse_ee(e, scratch); break; #ifdef EXTSTORE case LOGGER_EXT_WRITE_ENTRY: total = _logger_thread_parse_extw(e, scratch); break; #endif case LOGGER_ITEM_GET_ENTRY: total = _logger_thread_parse_ige(e, scratch); break; case LOGGER_ITEM_STORE_ENTRY: total = _logger_thread_parse_ise(e, scratch); break; } if (total >= LOGGER_PARSE_SCRATCH || total <= 0) { L_DEBUG("LOGGER: Failed to flatten log entry!\n"); return LOGGER_PARSE_ENTRY_FAILED; } else { *scratch_len = total; } return LOGGER_PARSE_ENTRY_OK; } /* Writes flattened entry to available watchers */ static void logger_thread_write_entry(logentry *e, struct logger_stats *ls, char *scratch, int scratch_len) { int x, total; /* Write the line into available watchers with matching flags */ for (x = 0; x < WATCHER_LIMIT; x++) { logger_watcher *w = watchers[x]; char *skip_scr = NULL; if (w == NULL || (e->eflags & w->eflags) == 0) continue; /* Avoid poll()'ing constantly when buffer is full by resetting a * flag periodically. */ while (!w->failed_flush && (skip_scr = (char *) bipbuf_request(w->buf, scratch_len + 128)) == NULL) { if (logger_thread_poll_watchers(0, x) <= 0) { L_DEBUG("LOGGER: Watcher had no free space for line of size (%d)\n", scratch_len + 128); w->failed_flush = true; } } if (w->failed_flush) { L_DEBUG("LOGGER: Fast skipped for watcher [%d] due to failed_flush\n", w->sfd); w->skipped++; ls->watcher_skipped++; continue; } if (w->skipped > 0) { total = snprintf(skip_scr, 128, "skipped=%llu\n", (unsigned long long) w->skipped); if (total >= 128 || total <= 0) { L_DEBUG("LOGGER: Failed to flatten skipped message into watcher [%d]\n", w->sfd); w->skipped++; ls->watcher_skipped++; continue; } bipbuf_push(w->buf, total); w->skipped = 0; } /* Can't fail because bipbuf_request succeeded. */ bipbuf_offer(w->buf, (unsigned char *) scratch, scratch_len); ls->watcher_sent++; } } /* Called with logger stack locked. * Releases every chunk associated with a watcher and closes the connection. * We can't presently send a connection back to the worker for further * processing. */ static void logger_thread_close_watcher(logger_watcher *w) { L_DEBUG("LOGGER: Closing dead watcher\n"); watchers[w->id] = NULL; sidethread_conn_close(w->c); watcher_count--; bipbuf_free(w->buf); free(w); logger_set_flags(); } /* Reads a particular worker thread's available bipbuf bytes. Parses each log * entry into the watcher buffers. */ static int logger_thread_read(logger *l, struct logger_stats *ls) { unsigned int size; unsigned int pos = 0; unsigned char *data; char scratch[LOGGER_PARSE_SCRATCH]; logentry *e; pthread_mutex_lock(&l->mutex); data = bipbuf_peek_all(l->buf, &size); pthread_mutex_unlock(&l->mutex); if (data == NULL) { return 0; } L_DEBUG("LOGGER: Got %d bytes from bipbuffer\n", size); /* parse buffer */ while (pos < size && watcher_count > 0) { enum logger_parse_entry_ret ret; int scratch_len = 0; e = (logentry *) (data + pos); ret = logger_thread_parse_entry(e, ls, scratch, &scratch_len); if (ret != LOGGER_PARSE_ENTRY_OK) { /* TODO: stats counter */ fprintf(stderr, "LOGGER: Failed to parse log entry\n"); } else { logger_thread_write_entry(e, ls, scratch, scratch_len); } pos += sizeof(logentry) + e->size + e->pad; } assert(pos <= size); pthread_mutex_lock(&l->mutex); data = bipbuf_poll(l->buf, size); ls->worker_written += l->written; ls->worker_dropped += l->dropped; l->written = 0; l->dropped = 0; pthread_mutex_unlock(&l->mutex); if (data == NULL) { fprintf(stderr, "LOGGER: unexpectedly couldn't advance buf pointer\n"); assert(0); } return size; /* maybe the count of objects iterated? */ } /* Since the event loop code isn't reusable without a refactor, and we have a * limited number of potential watchers, we run our own poll loop. * This calls poll() unnecessarily during write flushes, should be possible to * micro-optimize later. * * This flushes buffers attached to watchers, iterating through the bytes set * to each worker. Also checks for readability in case client connection was * closed. * * Allows a specific watcher to be flushed (if buf full) */ static int logger_thread_poll_watchers(int force_poll, int watcher) { int x; int nfd = 0; unsigned char *data; unsigned int data_size = 0; int flushed = 0; for (x = 0; x < WATCHER_LIMIT; x++) { logger_watcher *w = watchers[x]; if (w == NULL || (watcher != WATCHER_ALL && x != watcher)) continue; data = bipbuf_peek_all(w->buf, &data_size); if (data != NULL) { watchers_pollfds[nfd].fd = w->sfd; watchers_pollfds[nfd].events = POLLOUT; nfd++; } else if (force_poll) { watchers_pollfds[nfd].fd = w->sfd; watchers_pollfds[nfd].events = POLLIN; nfd++; } /* This gets set after a call to poll, and should be used to gate on * calling poll again. */ w->failed_flush = false; } if (nfd == 0) return 0; //L_DEBUG("LOGGER: calling poll() [data_size: %d]\n", data_size); int ret = poll(watchers_pollfds, nfd, 0); if (ret < 0) { perror("something failed with logger thread watcher fd polling"); return -1; } nfd = 0; for (x = 0; x < WATCHER_LIMIT; x++) { logger_watcher *w = watchers[x]; if (w == NULL) continue; data_size = 0; /* Early detection of a disconnect. Otherwise we have to wait until * the next write */ if (watchers_pollfds[nfd].revents & POLLIN) { char buf[1]; int res = ((conn*)w->c)->read(w->c, buf, 1); if (res == 0 || (res == -1 && (errno != EAGAIN && errno != EWOULDBLOCK))) { L_DEBUG("LOGGER: watcher closed remotely\n"); logger_thread_close_watcher(w); nfd++; continue; } } if ((data = bipbuf_peek_all(w->buf, &data_size)) != NULL) { if (watchers_pollfds[nfd].revents & (POLLHUP|POLLERR)) { L_DEBUG("LOGGER: watcher closed during poll() call\n"); logger_thread_close_watcher(w); } else if (watchers_pollfds[nfd].revents & POLLOUT) { int total = 0; /* We can write a bit. */ switch (w->t) { case LOGGER_WATCHER_STDERR: total = fwrite(data, 1, data_size, stderr); break; case LOGGER_WATCHER_CLIENT: total = ((conn*)w->c)->write(w->c, data, data_size); break; } L_DEBUG("LOGGER: poll() wrote %d to %d (data_size: %d) (bipbuf_used: %d)\n", total, w->sfd, data_size, bipbuf_used(w->buf)); if (total == -1) { if (errno != EAGAIN && errno != EWOULDBLOCK) { logger_thread_close_watcher(w); } L_DEBUG("LOGGER: watcher hit EAGAIN\n"); } else if (total == 0) { logger_thread_close_watcher(w); } else { bipbuf_poll(w->buf, total); flushed += total; } } } nfd++; } return flushed; } static void logger_thread_sum_stats(struct logger_stats *ls) { STATS_LOCK(); stats.log_worker_dropped += ls->worker_dropped; stats.log_worker_written += ls->worker_written; stats.log_watcher_skipped += ls->watcher_skipped; stats.log_watcher_sent += ls->watcher_sent; STATS_UNLOCK(); } #define MAX_LOGGER_SLEEP 1000000 #define MIN_LOGGER_SLEEP 1000 /* Primary logger thread routine */ static void *logger_thread(void *arg) { useconds_t to_sleep = MIN_LOGGER_SLEEP; L_DEBUG("LOGGER: Starting logger thread\n"); // TODO: If we ever have item references in the logger code, will need to // ensure everything is dequeued before stopping the thread. while (do_run_logger_thread) { int found_logs = 0; logger *l; struct logger_stats ls; memset(&ls, 0, sizeof(struct logger_stats)); /* only sleep if we're *above* the minimum */ if (to_sleep > MIN_LOGGER_SLEEP) usleep(to_sleep); /* Call function to iterate each logger. */ pthread_mutex_lock(&logger_stack_lock); for (l = logger_stack_head; l != NULL; l=l->next) { /* lock logger, call function to manipulate it */ found_logs += logger_thread_read(l, &ls); } logger_thread_poll_watchers(1, WATCHER_ALL); pthread_mutex_unlock(&logger_stack_lock); /* TODO: abstract into a function and share with lru_crawler */ if (!found_logs) { if (to_sleep < MAX_LOGGER_SLEEP) to_sleep += to_sleep / 8; if (to_sleep > MAX_LOGGER_SLEEP) to_sleep = MAX_LOGGER_SLEEP; } else { to_sleep /= 2; if (to_sleep < MIN_LOGGER_SLEEP) to_sleep = MIN_LOGGER_SLEEP; } logger_thread_sum_stats(&ls); } return NULL; } static int start_logger_thread(void) { int ret; do_run_logger_thread = 1; if ((ret = pthread_create(&logger_tid, NULL, logger_thread, NULL)) != 0) { fprintf(stderr, "Can't start logger thread: %s\n", strerror(ret)); return -1; } return 0; } static int stop_logger_thread(void) { do_run_logger_thread = 0; pthread_join(logger_tid, NULL); return 0; } /************************* * Public functions for submitting logs and starting loggers from workers. *************************/ /* Global logger thread start/init */ void logger_init(void) { /* TODO: auto destructor when threads exit */ /* TODO: error handling */ /* init stack for iterating loggers */ logger_stack_head = 0; logger_stack_tail = 0; pthread_key_create(&logger_key, NULL); if (start_logger_thread() != 0) { abort(); } /* This is what adding a STDERR watcher looks like. should replace old * "verbose" settings. */ //logger_add_watcher(NULL, 0); return; } void logger_stop(void) { stop_logger_thread(); } /* called *from* the thread using a logger. * initializes the per-thread bipbuf, links it into the list of loggers */ logger *logger_create(void) { L_DEBUG("LOGGER: Creating and linking new logger instance\n"); logger *l = calloc(1, sizeof(logger)); if (l == NULL) { return NULL; } l->buf = bipbuf_new(settings.logger_buf_size); if (l->buf == NULL) { free(l); return NULL; } l->entry_map = default_entries; pthread_mutex_init(&l->mutex, NULL); pthread_setspecific(logger_key, l); /* add to list of loggers */ logger_link_q(l); return l; } /* helpers for logger_log */ static void _logger_log_evictions(logentry *e, item *it) { struct logentry_eviction *le = (struct logentry_eviction *) e->data; le->exptime = (it->exptime > 0) ? (long long int)(it->exptime - current_time) : (long long int) -1; le->latime = current_time - it->time; le->it_flags = it->it_flags; le->nkey = it->nkey; le->clsid = ITEM_clsid(it); memcpy(le->key, ITEM_key(it), it->nkey); e->size = sizeof(struct logentry_eviction) + le->nkey; } #ifdef EXTSTORE /* TODO: When more logging endpoints are done and the extstore API has matured * more, this could be merged with above and print different types of * expulsion events. */ static void _logger_log_ext_write(logentry *e, item *it, uint8_t bucket) { struct logentry_ext_write *le = (struct logentry_ext_write *) e->data; le->exptime = (it->exptime > 0) ? (long long int)(it->exptime - current_time) : (long long int) -1; le->latime = current_time - it->time; le->it_flags = it->it_flags; le->nkey = it->nkey; le->clsid = ITEM_clsid(it); le->bucket = bucket; memcpy(le->key, ITEM_key(it), it->nkey); e->size = sizeof(struct logentry_ext_write) + le->nkey; } #endif /* 0 == nf, 1 == found. 2 == flushed. 3 == expired. * might be useful to store/print the flags an item has? * could also collapse this and above code into an "item status" struct. wait * for more endpoints to be written before making it generic, though. */ static void _logger_log_item_get(logentry *e, const int was_found, const char *key, const int nkey, const uint8_t clsid, const int sfd) { struct logentry_item_get *le = (struct logentry_item_get *) e->data; le->was_found = was_found; le->nkey = nkey; le->clsid = clsid; memcpy(le->key, key, nkey); le->sfd = sfd; e->size = sizeof(struct logentry_item_get) + nkey; } static void _logger_log_item_store(logentry *e, const enum store_item_type status, const int comm, char *key, const int nkey, rel_time_t ttl, const uint8_t clsid, int sfd) { struct logentry_item_store *le = (struct logentry_item_store *) e->data; le->status = status; le->cmd = comm; le->nkey = nkey; le->clsid = clsid; if (ttl != 0) { le->ttl = ttl - current_time; } else { le->ttl = 0; } memcpy(le->key, key, nkey); le->sfd = sfd; e->size = sizeof(struct logentry_item_store) + nkey; } /* Public function for logging an entry. * Tries to encapsulate as much of the formatting as possible to simplify the * caller's code. */ enum logger_ret_type logger_log(logger *l, const enum log_entry_type event, const void *entry, ...) { bipbuf_t *buf = l->buf; bool nospace = false; va_list ap; int total = 0; logentry *e; const entry_details *d = &l->entry_map[event]; int reqlen = d->reqlen; pthread_mutex_lock(&l->mutex); /* Request a maximum length of data to write to */ e = (logentry *) bipbuf_request(buf, (sizeof(logentry) + reqlen)); if (e == NULL) { pthread_mutex_unlock(&l->mutex); l->dropped++; return LOGGER_RET_NOSPACE; } e->event = d->subtype; e->pad = 0; e->gid = logger_get_gid(); /* TODO: Could pass this down as an argument now that we're using * LOGGER_LOG() macro. */ e->eflags = d->eflags; /* Noting time isn't optional. A feature may be added to avoid rendering * time and/or gid to a logger. */ gettimeofday(&e->tv, NULL); switch (d->subtype) { case LOGGER_TEXT_ENTRY: va_start(ap, entry); total = vsnprintf((char *) e->data, reqlen, d->format, ap); va_end(ap); if (total >= reqlen || total <= 0) { fprintf(stderr, "LOGGER: Failed to vsnprintf a text entry: (total) %d\n", total); break; } e->size = total + 1; /* null byte */ break; case LOGGER_EVICTION_ENTRY: _logger_log_evictions(e, (item *)entry); break; #ifdef EXTSTORE case LOGGER_EXT_WRITE_ENTRY: va_start(ap, entry); int ew_bucket = va_arg(ap, int); va_end(ap); _logger_log_ext_write(e, (item *)entry, ew_bucket); break; #endif case LOGGER_ITEM_GET_ENTRY: va_start(ap, entry); int was_found = va_arg(ap, int); char *key = va_arg(ap, char *); size_t nkey = va_arg(ap, size_t); uint8_t gclsid = va_arg(ap, int); int gsfd = va_arg(ap, int); _logger_log_item_get(e, was_found, key, nkey, gclsid, gsfd); va_end(ap); break; case LOGGER_ITEM_STORE_ENTRY: va_start(ap, entry); enum store_item_type status = va_arg(ap, enum store_item_type); int comm = va_arg(ap, int); char *skey = va_arg(ap, char *); size_t snkey = va_arg(ap, size_t); rel_time_t sttl = va_arg(ap, rel_time_t); uint8_t sclsid = va_arg(ap, int); int ssfd = va_arg(ap, int); _logger_log_item_store(e, status, comm, skey, snkey, sttl, sclsid, ssfd); va_end(ap); break; } #ifdef NEED_ALIGN /* Need to ensure *next* request is aligned. */ if (sizeof(logentry) + e->size % 8 != 0) { e->pad = 8 - (sizeof(logentry) + e->size % 8); } #endif /* Push pointer forward by the actual amount required */ if (bipbuf_push(buf, (sizeof(logentry) + e->size + e->pad)) == 0) { fprintf(stderr, "LOGGER: Failed to bipbuf push a text entry\n"); pthread_mutex_unlock(&l->mutex); return LOGGER_RET_ERR; } l->written++; L_DEBUG("LOGGER: Requested %d bytes, wrote %lu bytes\n", reqlen, (sizeof(logentry) + e->size)); pthread_mutex_unlock(&l->mutex); if (nospace) { return LOGGER_RET_NOSPACE; } else { return LOGGER_RET_OK; } } /* Passes a client connection socket from a primary worker thread to the * logger thread. Caller *must* event_del() the client before handing it over. * Presently there's no way to hand the client back to the worker thread. */ enum logger_add_watcher_ret logger_add_watcher(void *c, const int sfd, uint16_t f) { int x; logger_watcher *w = NULL; pthread_mutex_lock(&logger_stack_lock); if (watcher_count >= WATCHER_LIMIT) { pthread_mutex_unlock(&logger_stack_lock); return LOGGER_ADD_WATCHER_TOO_MANY; } for (x = 0; x < WATCHER_LIMIT-1; x++) { if (watchers[x] == NULL) break; } w = calloc(1, sizeof(logger_watcher)); if (w == NULL) { pthread_mutex_unlock(&logger_stack_lock); return LOGGER_ADD_WATCHER_FAILED; } w->c = c; w->sfd = sfd; if (sfd == 0 && c == NULL) { w->t = LOGGER_WATCHER_STDERR; } else { w->t = LOGGER_WATCHER_CLIENT; } w->id = x; w->eflags = f; w->buf = bipbuf_new(settings.logger_watcher_buf_size); if (w->buf == NULL) { free(w); pthread_mutex_unlock(&logger_stack_lock); return LOGGER_ADD_WATCHER_FAILED; } bipbuf_offer(w->buf, (unsigned char *) "OK\r\n", 4); watchers[x] = w; watcher_count++; /* Update what flags the global logs will watch */ logger_set_flags(); pthread_mutex_unlock(&logger_stack_lock); return LOGGER_ADD_WATCHER_OK; } memcached-1.5.22/freebsd_priv.c0000664000175000017500000000254213575022500013252 00000000000000#include #include #include #include #include #include #include "memcached.h" /* * dropping privileges is entering in capability mode * in FreeBSD vocabulary. */ void drop_privileges() { cap_rights_t wd, rd; if (cap_rights_init(&wd, CAP_WRITE, CAP_READ) == NULL) { fprintf(stderr, "cap_rights_init write protection failed: %s\n", strerror(errno)); exit(EXIT_FAILURE); } if (cap_rights_init(&rd, CAP_FCNTL, CAP_READ, CAP_EVENT) == NULL) { fprintf(stderr, "cap_rights_init read protection failed: %s\n", strerror(errno)); exit(EXIT_FAILURE); } if (cap_rights_limit(STDIN_FILENO, &rd) != 0) { fprintf(stderr, "cap_rights_limit stdin failed: %s\n", strerror(errno)); exit(EXIT_FAILURE); } if (cap_rights_limit(STDOUT_FILENO, &wd) != 0) { fprintf(stderr, "cap_rights_limit stdout failed: %s\n", strerror(errno)); exit(EXIT_FAILURE); } if (cap_rights_limit(STDERR_FILENO, &wd) != 0) { fprintf(stderr, "cap_rights_limit stderr failed: %s\n", strerror(errno)); exit(EXIT_FAILURE); } if (cap_enter() != 0) { fprintf(stderr, "cap_enter failed: %s\n", strerror(errno)); exit(EXIT_FAILURE); } } void setup_privilege_violations_handler(void) { // not needed } memcached-1.5.22/jenkins_hash.h0000664000175000017500000000032513461170555013256 00000000000000#ifndef JENKINS_HASH_H #define JENKINS_HASH_H #ifdef __cplusplus extern "C" { #endif uint32_t jenkins_hash(const void *key, size_t length); #ifdef __cplusplus } #endif #endif /* JENKINS_HASH_H */ memcached-1.5.22/slab_automove.c0000664000175000017500000001151313461170555013446 00000000000000/* Copyright 2017 Facebook. * * Use and distribution licensed under the BSD license. See * the LICENSE file for full text. */ /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #include "memcached.h" #include "slab_automove.h" #include #include #define MIN_PAGES_FOR_SOURCE 2 #define MIN_PAGES_FOR_RECLAIM 2.5 struct window_data { uint64_t age; uint64_t dirty; uint64_t evicted; }; typedef struct { struct window_data *window_data; uint32_t window_size; uint32_t window_cur; double max_age_ratio; item_stats_automove iam_before[MAX_NUMBER_OF_SLAB_CLASSES]; item_stats_automove iam_after[MAX_NUMBER_OF_SLAB_CLASSES]; slab_stats_automove sam_before[MAX_NUMBER_OF_SLAB_CLASSES]; slab_stats_automove sam_after[MAX_NUMBER_OF_SLAB_CLASSES]; } slab_automove; void *slab_automove_init(struct settings *settings) { uint32_t window_size = settings->slab_automove_window; double max_age_ratio = settings->slab_automove_ratio; slab_automove *a = calloc(1, sizeof(slab_automove)); if (a == NULL) return NULL; a->window_data = calloc(window_size * MAX_NUMBER_OF_SLAB_CLASSES, sizeof(struct window_data)); a->window_size = window_size; a->max_age_ratio = max_age_ratio; if (a->window_data == NULL) { free(a); return NULL; } // do a dry run to fill the before structs fill_item_stats_automove(a->iam_before); fill_slab_stats_automove(a->sam_before); return (void *)a; } void slab_automove_free(void *arg) { slab_automove *a = (slab_automove *)arg; free(a->window_data); free(a); } static void window_sum(struct window_data *wd, struct window_data *w, uint32_t size) { int x; for (x = 0; x < size; x++) { struct window_data *d = &wd[x]; w->age += d->age; w->dirty += d->dirty; w->evicted += d->evicted; } } // TODO: if oldest is dirty, find next oldest. // still need to base ratio off of absolute age void slab_automove_run(void *arg, int *src, int *dst) { slab_automove *a = (slab_automove *)arg; int n; struct window_data w_sum; int oldest = -1; uint64_t oldest_age = 0; int youngest = -1; uint64_t youngest_age = ~0; bool youngest_evicting = false; *src = -1; *dst = -1; // fill after structs fill_item_stats_automove(a->iam_after); fill_slab_stats_automove(a->sam_after); a->window_cur++; // iterate slabs for (n = POWER_SMALLEST; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) { int w_offset = n * a->window_size; struct window_data *wd = &a->window_data[w_offset + (a->window_cur % a->window_size)]; memset(wd, 0, sizeof(struct window_data)); // summarize the window-up-to-now. memset(&w_sum, 0, sizeof(struct window_data)); window_sum(&a->window_data[w_offset], &w_sum, a->window_size); // if page delta, or evicted delta, mark window dirty // (or outofmemory) if (a->iam_after[n].evicted - a->iam_before[n].evicted > 0 || a->iam_after[n].outofmemory - a->iam_before[n].outofmemory > 0) { wd->evicted = 1; wd->dirty = 1; } if (a->sam_after[n].total_pages - a->sam_before[n].total_pages > 0) { wd->dirty = 1; } // set age into window wd->age = a->iam_after[n].age; // grab age as average of window total uint64_t age = w_sum.age / a->window_size; // if > N free chunks and not dirty, make decision. if (a->sam_after[n].free_chunks > a->sam_after[n].chunks_per_page * MIN_PAGES_FOR_RECLAIM) { if (w_sum.dirty == 0) { *src = n; *dst = 0; break; } } // if oldest and have enough pages, is oldest if (age > oldest_age && a->sam_after[n].total_pages > MIN_PAGES_FOR_SOURCE) { oldest = n; oldest_age = age; } // grab evicted count from window // if > half the window and youngest, mark as youngest if (age < youngest_age && w_sum.evicted > a->window_size / 2) { youngest = n; youngest_age = age; youngest_evicting = wd->evicted ? true : false; } } memcpy(a->iam_before, a->iam_after, sizeof(item_stats_automove) * MAX_NUMBER_OF_SLAB_CLASSES); memcpy(a->sam_before, a->sam_after, sizeof(slab_stats_automove) * MAX_NUMBER_OF_SLAB_CLASSES); // if we have a youngest and oldest, and oldest is outside the ratio, // also, only make decisions if window has filled once. if (youngest != -1 && oldest != -1 && a->window_cur > a->window_size) { if (youngest_age < ((double)oldest_age * a->max_age_ratio) && youngest_evicting) { *src = oldest; *dst = youngest; } } return; } memcached-1.5.22/timedrun.c0000664000175000017500000000470213575022500012427 00000000000000#include #include #include #include #include #include #include volatile sig_atomic_t caught_sig = 0; static void signal_handler(int which) { caught_sig = which; } static int wait_for_process(pid_t pid) { int rv = EX_SOFTWARE; int status = 0; int i = 0; struct sigaction sig_handler; sig_handler.sa_handler = signal_handler; sig_handler.sa_flags = 0; sigaction(SIGALRM, &sig_handler, NULL); sigaction(SIGHUP, &sig_handler, NULL); sigaction(SIGINT, &sig_handler, NULL); sigaction(SIGUSR1, &sig_handler, NULL); sigaction(SIGTERM, &sig_handler, NULL); sigaction(SIGPIPE, &sig_handler, NULL); /* Loop forever waiting for the process to quit */ for (i = 0; ;i++) { pid_t p = waitpid(pid, &status, 0); if (p == pid) { /* child exited. Let's get out of here */ rv = WIFEXITED(status) ? WEXITSTATUS(status) : (0x80 | WTERMSIG(status)); break; } else { int sig = 0; switch (i) { case 0: /* On the first iteration, pass the signal through */ sig = caught_sig > 0 ? caught_sig : SIGTERM; if (caught_sig == SIGALRM) { fprintf(stderr, "Timeout.. killing the process\n"); } break; case 1: sig = SIGTERM; break; default: sig = SIGKILL; break; } if (kill(pid, sig) < 0) { /* Kill failed. Must have lost the process. :/ */ perror("lost child when trying to kill"); } /* Wait up to 5 seconds for the pid */ alarm(5); } } return rv; } static int spawn_and_wait(char **argv) { int rv = EX_SOFTWARE; pid_t pid = fork(); switch (pid) { case -1: perror("fork"); rv = EX_OSERR; break; /* NOTREACHED */ case 0: execvp(argv[0], argv); perror("exec"); rv = EX_SOFTWARE; break; /* NOTREACHED */ default: rv = wait_for_process(pid); } return rv; } int main(int argc, char **argv) { int naptime = 0; assert(argc > 2); naptime = atoi(argv[1]); assert(naptime > 0 && naptime < 1800); alarm(naptime); return spawn_and_wait(argv+2); } memcached-1.5.22/slab_automove.h0000664000175000017500000000107013461170555013450 00000000000000#ifndef SLAB_AUTOMOVE_H #define SLAB_AUTOMOVE_H /* default automove functions */ void *slab_automove_init(struct settings *settings); void slab_automove_free(void *arg); void slab_automove_run(void *arg, int *src, int *dst); typedef void *(*slab_automove_init_func)(struct settings *settings); typedef void (*slab_automove_free_func)(void *arg); typedef void (*slab_automove_run_func)(void *arg, int *src, int *dst); typedef struct { slab_automove_init_func init; slab_automove_free_func free; slab_automove_run_func run; } slab_automove_reg_t; #endif memcached-1.5.22/slabs.c0000664000175000017500000013063213615376546011727 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ /* * Slabs memory allocation, based on powers-of-N. Slabs are up to 1MB in size * and are divided into chunks. The chunk sizes start off at the size of the * "item" structure plus space for a small key and value. They increase by * a multiplier factor from there, up to half the maximum slab size. The last * slab size is always 1MB, since that's the maximum item size allowed by the * memcached protocol. */ #include "memcached.h" #include #include #include #include #include #include #include #include #include #include #include #include #include //#define DEBUG_SLAB_MOVER /* powers-of-N allocation structures */ typedef struct { unsigned int size; /* sizes of items */ unsigned int perslab; /* how many items per slab */ void *slots; /* list of item ptrs */ unsigned int sl_curr; /* total free items in list */ unsigned int slabs; /* how many slabs were allocated for this class */ void **slab_list; /* array of slab pointers */ unsigned int list_size; /* size of prev array */ } slabclass_t; static slabclass_t slabclass[MAX_NUMBER_OF_SLAB_CLASSES]; static size_t mem_limit = 0; static size_t mem_malloced = 0; /* If the memory limit has been hit once. Used as a hint to decide when to * early-wake the LRU maintenance thread */ static bool mem_limit_reached = false; static int power_largest; static void *mem_base = NULL; static void *mem_current = NULL; static size_t mem_avail = 0; #ifdef EXTSTORE static void *storage = NULL; #endif /** * Access to the slab allocator is protected by this lock */ static pthread_mutex_t slabs_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t slabs_rebalance_lock = PTHREAD_MUTEX_INITIALIZER; /* * Forward Declarations */ static int grow_slab_list (const unsigned int id); static int do_slabs_newslab(const unsigned int id); static void *memory_allocate(size_t size); static void do_slabs_free(void *ptr, const size_t size, unsigned int id); /* Preallocate as many slab pages as possible (called from slabs_init) on start-up, so users don't get confused out-of-memory errors when they do have free (in-slab) space, but no space to make new slabs. if maxslabs is 18 (POWER_LARGEST - POWER_SMALLEST + 1), then all slab types can be made. if max memory is less than 18 MB, only the smaller ones will be made. */ static void slabs_preallocate (const unsigned int maxslabs); #ifdef EXTSTORE void slabs_set_storage(void *arg) { storage = arg; } #endif /* * Figures out which slab class (chunk size) is required to store an item of * a given size. * * Given object size, return id to use when allocating/freeing memory for object * 0 means error: can't store such a large object */ unsigned int slabs_clsid(const size_t size) { int res = POWER_SMALLEST; if (size == 0 || size > settings.item_size_max) return 0; while (size > slabclass[res].size) if (res++ == power_largest) /* won't fit in the biggest slab */ return power_largest; return res; } unsigned int slabs_size(const int clsid) { return slabclass[clsid].size; } // TODO: could this work with the restartable memory? // Docs say hugepages only work with private shm allocs. /* Function split out for better error path handling */ static void * alloc_large_chunk(const size_t limit) { void *ptr = NULL; #if defined(__linux__) && defined(MADV_HUGEPAGE) size_t pagesize = 0; FILE *fp; int ret; /* Get the size of huge pages */ fp = fopen("/proc/meminfo", "r"); if (fp != NULL) { char buf[64]; while ((fgets(buf, sizeof(buf), fp))) if (!strncmp(buf, "Hugepagesize:", 13)) { ret = sscanf(buf + 13, "%zu\n", &pagesize); /* meminfo huge page size is in KiBs */ pagesize <<= 10; } fclose(fp); } if (!pagesize) { fprintf(stderr, "Failed to get supported huge page size\n"); return NULL; } if (settings.verbose > 1) fprintf(stderr, "huge page size: %zu\n", pagesize); /* This works because glibc simply uses mmap when the alignment is * above a certain limit. */ ret = posix_memalign(&ptr, pagesize, limit); if (ret != 0) { fprintf(stderr, "Failed to get aligned memory chunk: %d\n", ret); return NULL; } ret = madvise(ptr, limit, MADV_HUGEPAGE); if (ret < 0) { fprintf(stderr, "Failed to set transparent hugepage hint: %d\n", ret); free(ptr); ptr = NULL; } #elif defined(__FreeBSD__) size_t align = (sizeof(size_t) * 8 - (__builtin_clzl(4095))); ptr = mmap(NULL, limit, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON | MAP_ALIGNED(align) | MAP_ALIGNED_SUPER, -1, 0); if (ptr == MAP_FAILED) { fprintf(stderr, "Failed to set super pages\n"); ptr = NULL; } #else ptr = malloc(limit); #endif return ptr; } unsigned int slabs_fixup(char *chunk, const int border) { slabclass_t *p; item *it = (item *)chunk; int id = ITEM_clsid(it); // memory isn't used yet. shunt to global pool. // (which must be 0) if (id == 0) { //assert(border == 0); p = &slabclass[0]; grow_slab_list(0); p->slab_list[p->slabs++] = (char*)chunk; return -1; } p = &slabclass[id]; // if we're on a page border, add the slab to slab class if (border == 0) { grow_slab_list(id); p->slab_list[p->slabs++] = chunk; } // increase free count if ITEM_SLABBED if (it->it_flags == ITEM_SLABBED) { // if ITEM_SLABBED re-stack on freelist. // don't have to run pointer fixups. it->prev = 0; it->next = p->slots; if (it->next) it->next->prev = it; p->slots = it; p->sl_curr++; //fprintf(stderr, "replacing into freelist\n"); } return p->size; } /** * Determines the chunk sizes and initializes the slab class descriptors * accordingly. */ void slabs_init(const size_t limit, const double factor, const bool prealloc, const uint32_t *slab_sizes, void *mem_base_external, bool reuse_mem) { int i = POWER_SMALLEST - 1; unsigned int size = sizeof(item) + settings.chunk_size; /* Some platforms use runtime transparent hugepages. If for any reason * the initial allocation fails, the required settings do not persist * for remaining allocations. As such it makes little sense to do slab * preallocation. */ bool __attribute__ ((unused)) do_slab_prealloc = false; mem_limit = limit; if (prealloc && mem_base_external == NULL) { mem_base = alloc_large_chunk(mem_limit); if (mem_base) { do_slab_prealloc = true; mem_current = mem_base; mem_avail = mem_limit; } else { fprintf(stderr, "Warning: Failed to allocate requested memory in" " one large chunk.\nWill allocate in smaller chunks\n"); } } else if (prealloc && mem_base_external != NULL) { // Can't (yet) mix hugepages with mmap allocations, so separate the // logic from above. Reusable memory also force-preallocates memory // pages into the global pool, which requires turning mem_* variables. do_slab_prealloc = true; mem_base = mem_base_external; // _current shouldn't be used in this case, but we set it to where it // should be anyway. if (reuse_mem) { mem_current = ((char*)mem_base) + mem_limit; mem_avail = 0; } else { mem_current = mem_base; mem_avail = mem_limit; } } memset(slabclass, 0, sizeof(slabclass)); while (++i < MAX_NUMBER_OF_SLAB_CLASSES-1) { if (slab_sizes != NULL) { if (slab_sizes[i-1] == 0) break; size = slab_sizes[i-1]; } else if (size >= settings.slab_chunk_size_max / factor) { break; } /* Make sure items are always n-byte aligned */ if (size % CHUNK_ALIGN_BYTES) size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES); slabclass[i].size = size; slabclass[i].perslab = settings.slab_page_size / slabclass[i].size; if (slab_sizes == NULL) size *= factor; if (settings.verbose > 1) { fprintf(stderr, "slab class %3d: chunk size %9u perslab %7u\n", i, slabclass[i].size, slabclass[i].perslab); } } power_largest = i; slabclass[power_largest].size = settings.slab_chunk_size_max; slabclass[power_largest].perslab = settings.slab_page_size / settings.slab_chunk_size_max; if (settings.verbose > 1) { fprintf(stderr, "slab class %3d: chunk size %9u perslab %7u\n", i, slabclass[i].size, slabclass[i].perslab); } /* for the test suite: faking of how much we've already malloc'd */ { char *t_initial_malloc = getenv("T_MEMD_INITIAL_MALLOC"); if (t_initial_malloc) { mem_malloced = (size_t)atol(t_initial_malloc); } } if (do_slab_prealloc) { if (!reuse_mem) { slabs_preallocate(power_largest); } } } void slabs_prefill_global(void) { void *ptr; slabclass_t *p = &slabclass[0]; int len = settings.slab_page_size; while (mem_malloced < mem_limit && (ptr = memory_allocate(len)) != NULL) { grow_slab_list(0); p->slab_list[p->slabs++] = ptr; } mem_limit_reached = true; } static void slabs_preallocate (const unsigned int maxslabs) { int i; unsigned int prealloc = 0; /* pre-allocate a 1MB slab in every size class so people don't get confused by non-intuitive "SERVER_ERROR out of memory" messages. this is the most common question on the mailing list. if you really don't want this, you can rebuild without these three lines. */ for (i = POWER_SMALLEST; i < MAX_NUMBER_OF_SLAB_CLASSES; i++) { if (++prealloc > maxslabs) break; if (do_slabs_newslab(i) == 0) { fprintf(stderr, "Error while preallocating slab memory!\n" "If using -L or other prealloc options, max memory must be " "at least %d megabytes.\n", power_largest); exit(1); } } } static int grow_slab_list (const unsigned int id) { slabclass_t *p = &slabclass[id]; if (p->slabs == p->list_size) { size_t new_size = (p->list_size != 0) ? p->list_size * 2 : 16; void *new_list = realloc(p->slab_list, new_size * sizeof(void *)); if (new_list == 0) return 0; p->list_size = new_size; p->slab_list = new_list; } return 1; } static void split_slab_page_into_freelist(char *ptr, const unsigned int id) { slabclass_t *p = &slabclass[id]; int x; for (x = 0; x < p->perslab; x++) { do_slabs_free(ptr, 0, id); ptr += p->size; } } /* Fast FIFO queue */ static void *get_page_from_global_pool(void) { slabclass_t *p = &slabclass[SLAB_GLOBAL_PAGE_POOL]; if (p->slabs < 1) { return NULL; } char *ret = p->slab_list[p->slabs - 1]; p->slabs--; return ret; } static int do_slabs_newslab(const unsigned int id) { slabclass_t *p = &slabclass[id]; slabclass_t *g = &slabclass[SLAB_GLOBAL_PAGE_POOL]; int len = (settings.slab_reassign || settings.slab_chunk_size_max != settings.slab_page_size) ? settings.slab_page_size : p->size * p->perslab; char *ptr; if ((mem_limit && mem_malloced + len > mem_limit && p->slabs > 0 && g->slabs == 0)) { mem_limit_reached = true; MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED(id); return 0; } if ((grow_slab_list(id) == 0) || (((ptr = get_page_from_global_pool()) == NULL) && ((ptr = memory_allocate((size_t)len)) == 0))) { MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED(id); return 0; } #if !defined(__FreeBSD__) memset(ptr, 0, (size_t)len); #endif split_slab_page_into_freelist(ptr, id); p->slab_list[p->slabs++] = ptr; MEMCACHED_SLABS_SLABCLASS_ALLOCATE(id); return 1; } /*@null@*/ static void *do_slabs_alloc(const size_t size, unsigned int id, unsigned int flags) { slabclass_t *p; void *ret = NULL; item *it = NULL; if (id < POWER_SMALLEST || id > power_largest) { MEMCACHED_SLABS_ALLOCATE_FAILED(size, 0); return NULL; } p = &slabclass[id]; assert(p->sl_curr == 0 || (((item *)p->slots)->it_flags & ITEM_SLABBED)); assert(size <= p->size); /* fail unless we have space at the end of a recently allocated page, we have something on our freelist, or we could allocate a new page */ if (p->sl_curr == 0 && flags != SLABS_ALLOC_NO_NEWPAGE) { do_slabs_newslab(id); } if (p->sl_curr != 0) { /* return off our freelist */ it = (item *)p->slots; p->slots = it->next; if (it->next) it->next->prev = 0; /* Kill flag and initialize refcount here for lock safety in slab * mover's freeness detection. */ it->it_flags &= ~ITEM_SLABBED; it->refcount = 1; p->sl_curr--; ret = (void *)it; } else { ret = NULL; } if (ret) { MEMCACHED_SLABS_ALLOCATE(size, id, p->size, ret); } else { MEMCACHED_SLABS_ALLOCATE_FAILED(size, id); } return ret; } static void do_slabs_free_chunked(item *it, const size_t size) { item_chunk *chunk = (item_chunk *) ITEM_schunk(it); slabclass_t *p; it->it_flags = ITEM_SLABBED; // FIXME: refresh on how this works? //it->slabs_clsid = 0; it->prev = 0; // header object's original classid is stored in chunk. p = &slabclass[chunk->orig_clsid]; if (chunk->next) { chunk = chunk->next; chunk->prev = 0; } else { // header with no attached chunk chunk = NULL; } // return the header object. // TODO: This is in three places, here and in do_slabs_free(). it->prev = 0; it->next = p->slots; if (it->next) it->next->prev = it; p->slots = it; p->sl_curr++; item_chunk *next_chunk; while (chunk) { assert(chunk->it_flags == ITEM_CHUNK); chunk->it_flags = ITEM_SLABBED; p = &slabclass[chunk->slabs_clsid]; next_chunk = chunk->next; chunk->prev = 0; chunk->next = p->slots; if (chunk->next) chunk->next->prev = chunk; p->slots = chunk; p->sl_curr++; chunk = next_chunk; } return; } static void do_slabs_free(void *ptr, const size_t size, unsigned int id) { slabclass_t *p; item *it; assert(id >= POWER_SMALLEST && id <= power_largest); if (id < POWER_SMALLEST || id > power_largest) return; MEMCACHED_SLABS_FREE(size, id, ptr); p = &slabclass[id]; it = (item *)ptr; if ((it->it_flags & ITEM_CHUNKED) == 0) { it->it_flags = ITEM_SLABBED; it->slabs_clsid = id; it->prev = 0; it->next = p->slots; if (it->next) it->next->prev = it; p->slots = it; p->sl_curr++; } else { do_slabs_free_chunked(it, size); } return; } /* With refactoring of the various stats code the automover won't need a * custom function here. */ void fill_slab_stats_automove(slab_stats_automove *am) { int n; pthread_mutex_lock(&slabs_lock); for (n = 0; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) { slabclass_t *p = &slabclass[n]; slab_stats_automove *cur = &am[n]; cur->chunks_per_page = p->perslab; cur->free_chunks = p->sl_curr; cur->total_pages = p->slabs; cur->chunk_size = p->size; } pthread_mutex_unlock(&slabs_lock); } /* TODO: slabs_available_chunks should grow up to encompass this. * mem_flag is redundant with the other function. */ unsigned int global_page_pool_size(bool *mem_flag) { unsigned int ret = 0; pthread_mutex_lock(&slabs_lock); if (mem_flag != NULL) *mem_flag = mem_malloced >= mem_limit ? true : false; ret = slabclass[SLAB_GLOBAL_PAGE_POOL].slabs; pthread_mutex_unlock(&slabs_lock); return ret; } /*@null@*/ static void do_slabs_stats(ADD_STAT add_stats, void *c) { int i, total; /* Get the per-thread stats which contain some interesting aggregates */ struct thread_stats thread_stats; threadlocal_stats_aggregate(&thread_stats); total = 0; for(i = POWER_SMALLEST; i <= power_largest; i++) { slabclass_t *p = &slabclass[i]; if (p->slabs != 0) { uint32_t perslab, slabs; slabs = p->slabs; perslab = p->perslab; char key_str[STAT_KEY_LEN]; char val_str[STAT_VAL_LEN]; int klen = 0, vlen = 0; APPEND_NUM_STAT(i, "chunk_size", "%u", p->size); APPEND_NUM_STAT(i, "chunks_per_page", "%u", perslab); APPEND_NUM_STAT(i, "total_pages", "%u", slabs); APPEND_NUM_STAT(i, "total_chunks", "%u", slabs * perslab); APPEND_NUM_STAT(i, "used_chunks", "%u", slabs*perslab - p->sl_curr); APPEND_NUM_STAT(i, "free_chunks", "%u", p->sl_curr); /* Stat is dead, but displaying zero instead of removing it. */ APPEND_NUM_STAT(i, "free_chunks_end", "%u", 0); APPEND_NUM_STAT(i, "get_hits", "%llu", (unsigned long long)thread_stats.slab_stats[i].get_hits); APPEND_NUM_STAT(i, "cmd_set", "%llu", (unsigned long long)thread_stats.slab_stats[i].set_cmds); APPEND_NUM_STAT(i, "delete_hits", "%llu", (unsigned long long)thread_stats.slab_stats[i].delete_hits); APPEND_NUM_STAT(i, "incr_hits", "%llu", (unsigned long long)thread_stats.slab_stats[i].incr_hits); APPEND_NUM_STAT(i, "decr_hits", "%llu", (unsigned long long)thread_stats.slab_stats[i].decr_hits); APPEND_NUM_STAT(i, "cas_hits", "%llu", (unsigned long long)thread_stats.slab_stats[i].cas_hits); APPEND_NUM_STAT(i, "cas_badval", "%llu", (unsigned long long)thread_stats.slab_stats[i].cas_badval); APPEND_NUM_STAT(i, "touch_hits", "%llu", (unsigned long long)thread_stats.slab_stats[i].touch_hits); total++; } } /* add overall slab stats and append terminator */ APPEND_STAT("active_slabs", "%d", total); APPEND_STAT("total_malloced", "%llu", (unsigned long long)mem_malloced); add_stats(NULL, 0, NULL, 0, c); } static void *memory_allocate(size_t size) { void *ret; if (mem_base == NULL) { /* We are not using a preallocated large memory chunk */ ret = malloc(size); } else { ret = mem_current; if (size > mem_avail) { return NULL; } /* mem_current pointer _must_ be aligned!!! */ if (size % CHUNK_ALIGN_BYTES) { size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES); } mem_current = ((char*)mem_current) + size; if (size < mem_avail) { mem_avail -= size; } else { mem_avail = 0; } } mem_malloced += size; return ret; } /* Must only be used if all pages are item_size_max */ static void memory_release() { void *p = NULL; if (mem_base != NULL) return; if (!settings.slab_reassign) return; while (mem_malloced > mem_limit && (p = get_page_from_global_pool()) != NULL) { free(p); mem_malloced -= settings.slab_page_size; } } void *slabs_alloc(size_t size, unsigned int id, unsigned int flags) { void *ret; pthread_mutex_lock(&slabs_lock); ret = do_slabs_alloc(size, id, flags); pthread_mutex_unlock(&slabs_lock); return ret; } void slabs_free(void *ptr, size_t size, unsigned int id) { pthread_mutex_lock(&slabs_lock); do_slabs_free(ptr, size, id); pthread_mutex_unlock(&slabs_lock); } void slabs_stats(ADD_STAT add_stats, void *c) { pthread_mutex_lock(&slabs_lock); do_slabs_stats(add_stats, c); pthread_mutex_unlock(&slabs_lock); } static bool do_slabs_adjust_mem_limit(size_t new_mem_limit) { /* Cannot adjust memory limit at runtime if prealloc'ed */ if (mem_base != NULL) return false; settings.maxbytes = new_mem_limit; mem_limit = new_mem_limit; mem_limit_reached = false; /* Will reset on next alloc */ memory_release(); /* free what might already be in the global pool */ return true; } bool slabs_adjust_mem_limit(size_t new_mem_limit) { bool ret; pthread_mutex_lock(&slabs_lock); ret = do_slabs_adjust_mem_limit(new_mem_limit); pthread_mutex_unlock(&slabs_lock); return ret; } unsigned int slabs_available_chunks(const unsigned int id, bool *mem_flag, unsigned int *chunks_perslab) { unsigned int ret; slabclass_t *p; pthread_mutex_lock(&slabs_lock); p = &slabclass[id]; ret = p->sl_curr; if (mem_flag != NULL) *mem_flag = mem_malloced >= mem_limit ? true : false; if (chunks_perslab != NULL) *chunks_perslab = p->perslab; pthread_mutex_unlock(&slabs_lock); return ret; } /* The slabber system could avoid needing to understand much, if anything, * about items if callbacks were strategically used. Due to how the slab mover * works, certain flag bits can only be adjusted while holding the slabs lock. * Using these functions, isolate sections of code needing this and turn them * into callbacks when an interface becomes more obvious. */ void slabs_mlock(void) { pthread_mutex_lock(&slabs_lock); } void slabs_munlock(void) { pthread_mutex_unlock(&slabs_lock); } static pthread_cond_t slab_rebalance_cond = PTHREAD_COND_INITIALIZER; static volatile int do_run_slab_rebalance_thread = 1; static int slab_rebalance_start(void) { slabclass_t *s_cls; int no_go = 0; pthread_mutex_lock(&slabs_lock); if (slab_rebal.s_clsid < SLAB_GLOBAL_PAGE_POOL || slab_rebal.s_clsid > power_largest || slab_rebal.d_clsid < SLAB_GLOBAL_PAGE_POOL || slab_rebal.d_clsid > power_largest || slab_rebal.s_clsid == slab_rebal.d_clsid) no_go = -2; s_cls = &slabclass[slab_rebal.s_clsid]; if (!grow_slab_list(slab_rebal.d_clsid)) { no_go = -1; } if (s_cls->slabs < 2) no_go = -3; if (no_go != 0) { pthread_mutex_unlock(&slabs_lock); return no_go; /* Should use a wrapper function... */ } /* Always kill the first available slab page as it is most likely to * contain the oldest items */ slab_rebal.slab_start = s_cls->slab_list[0]; slab_rebal.slab_end = (char *)slab_rebal.slab_start + (s_cls->size * s_cls->perslab); slab_rebal.slab_pos = slab_rebal.slab_start; slab_rebal.done = 0; // Don't need to do chunk move work if page is in global pool. if (slab_rebal.s_clsid == SLAB_GLOBAL_PAGE_POOL) { slab_rebal.done = 1; } // Bit-vector to keep track of completed chunks slab_rebal.completed = (uint8_t*)calloc(s_cls->perslab,sizeof(uint8_t)); slab_rebalance_signal = 2; if (settings.verbose > 1) { fprintf(stderr, "Started a slab rebalance\n"); } pthread_mutex_unlock(&slabs_lock); STATS_LOCK(); stats_state.slab_reassign_running = true; STATS_UNLOCK(); return 0; } /* CALLED WITH slabs_lock HELD */ static void *slab_rebalance_alloc(const size_t size, unsigned int id) { slabclass_t *s_cls; s_cls = &slabclass[slab_rebal.s_clsid]; int x; item *new_it = NULL; for (x = 0; x < s_cls->perslab; x++) { new_it = do_slabs_alloc(size, id, SLABS_ALLOC_NO_NEWPAGE); /* check that memory isn't within the range to clear */ if (new_it == NULL) { break; } if ((void *)new_it >= slab_rebal.slab_start && (void *)new_it < slab_rebal.slab_end) { /* Pulled something we intend to free. Mark it as freed since * we've already done the work of unlinking it from the freelist. */ new_it->refcount = 0; new_it->it_flags = ITEM_SLABBED|ITEM_FETCHED; #ifdef DEBUG_SLAB_MOVER memcpy(ITEM_key(new_it), "deadbeef", 8); #endif new_it = NULL; slab_rebal.inline_reclaim++; } else { break; } } return new_it; } /* CALLED WITH slabs_lock HELD */ /* detaches item/chunk from freelist. */ static void slab_rebalance_cut_free(slabclass_t *s_cls, item *it) { /* Ensure this was on the freelist and nothing else. */ assert(it->it_flags == ITEM_SLABBED); if (s_cls->slots == it) { s_cls->slots = it->next; } if (it->next) it->next->prev = it->prev; if (it->prev) it->prev->next = it->next; s_cls->sl_curr--; } enum move_status { MOVE_PASS=0, MOVE_FROM_SLAB, MOVE_FROM_LRU, MOVE_BUSY, MOVE_LOCKED }; #define SLAB_MOVE_MAX_LOOPS 1000 /* refcount == 0 is safe since nobody can incr while item_lock is held. * refcount != 0 is impossible since flags/etc can be modified in other * threads. instead, note we found a busy one and bail. logic in do_item_get * will prevent busy items from continuing to be busy * NOTE: This is checking it_flags outside of an item lock. I believe this * works since it_flags is 8 bits, and we're only ever comparing a single bit * regardless. ITEM_SLABBED bit will always be correct since we're holding the * lock which modifies that bit. ITEM_LINKED won't exist if we're between an * item having ITEM_SLABBED removed, and the key hasn't been added to the item * yet. The memory barrier from the slabs lock should order the key write and the * flags to the item? * If ITEM_LINKED did exist and was just removed, but we still see it, that's * still safe since it will have a valid key, which we then lock, and then * recheck everything. * This may not be safe on all platforms; If not, slabs_alloc() will need to * seed the item key while holding slabs_lock. */ static int slab_rebalance_move(void) { slabclass_t *s_cls; int was_busy = 0; int refcount = 0; uint32_t hv; void *hold_lock; enum move_status status = MOVE_PASS; s_cls = &slabclass[slab_rebal.s_clsid]; // the offset to check if completed or not int offset = ((char*)slab_rebal.slab_pos-(char*)slab_rebal.slab_start)/(s_cls->size); // skip acquiring the slabs lock for items we've already fully processed. if (slab_rebal.completed[offset] == 0) { pthread_mutex_lock(&slabs_lock); hv = 0; hold_lock = NULL; item *it = slab_rebal.slab_pos; item_chunk *ch = NULL; status = MOVE_PASS; if (it->it_flags & ITEM_CHUNK) { /* This chunk is a chained part of a larger item. */ ch = (item_chunk *) it; /* Instead, we use the head chunk to find the item and effectively * lock the entire structure. If a chunk has ITEM_CHUNK flag, its * head cannot be slabbed, so the normal routine is safe. */ it = ch->head; assert(it->it_flags & ITEM_CHUNKED); } /* ITEM_FETCHED when ITEM_SLABBED is overloaded to mean we've cleared * the chunk for move. Only these two flags should exist. */ if (it->it_flags != (ITEM_SLABBED|ITEM_FETCHED)) { /* ITEM_SLABBED can only be added/removed under the slabs_lock */ if (it->it_flags & ITEM_SLABBED) { assert(ch == NULL); slab_rebalance_cut_free(s_cls, it); status = MOVE_FROM_SLAB; } else if ((it->it_flags & ITEM_LINKED) != 0) { /* If it doesn't have ITEM_SLABBED, the item could be in any * state on its way to being freed or written to. If no * ITEM_SLABBED, but it's had ITEM_LINKED, it must be active * and have the key written to it already. */ hv = hash(ITEM_key(it), it->nkey); if ((hold_lock = item_trylock(hv)) == NULL) { status = MOVE_LOCKED; } else { bool is_linked = (it->it_flags & ITEM_LINKED); refcount = refcount_incr(it); if (refcount == 2) { /* item is linked but not busy */ /* Double check ITEM_LINKED flag here, since we're * past a memory barrier from the mutex. */ if (is_linked) { status = MOVE_FROM_LRU; } else { /* refcount == 1 + !ITEM_LINKED means the item is being * uploaded to, or was just unlinked but hasn't been freed * yet. Let it bleed off on its own and try again later */ status = MOVE_BUSY; } } else if (refcount > 2 && is_linked) { // TODO: Mark items for delete/rescue and process // outside of the main loop. if (slab_rebal.busy_loops > SLAB_MOVE_MAX_LOOPS) { slab_rebal.busy_deletes++; // Only safe to hold slabs lock because refcount // can't drop to 0 until we release item lock. STORAGE_delete(storage, it); pthread_mutex_unlock(&slabs_lock); do_item_unlink(it, hv); pthread_mutex_lock(&slabs_lock); } status = MOVE_BUSY; } else { if (settings.verbose > 2) { fprintf(stderr, "Slab reassign hit a busy item: refcount: %d (%d -> %d)\n", it->refcount, slab_rebal.s_clsid, slab_rebal.d_clsid); } status = MOVE_BUSY; } /* Item lock must be held while modifying refcount */ if (status == MOVE_BUSY) { refcount_decr(it); item_trylock_unlock(hold_lock); } } } else { /* See above comment. No ITEM_SLABBED or ITEM_LINKED. Mark * busy and wait for item to complete its upload. */ status = MOVE_BUSY; } } int save_item = 0; item *new_it = NULL; size_t ntotal = 0; switch (status) { case MOVE_FROM_LRU: /* Lock order is LRU locks -> slabs_lock. unlink uses LRU lock. * We only need to hold the slabs_lock while initially looking * at an item, and at this point we have an exclusive refcount * (2) + the item is locked. Drop slabs lock, drop item to * refcount 1 (just our own, then fall through and wipe it */ /* Check if expired or flushed */ ntotal = ITEM_ntotal(it); #ifdef EXTSTORE if (it->it_flags & ITEM_HDR) { ntotal = (ntotal - it->nbytes) + sizeof(item_hdr); } #endif /* REQUIRES slabs_lock: CHECK FOR cls->sl_curr > 0 */ if (ch == NULL && (it->it_flags & ITEM_CHUNKED)) { /* Chunked should be identical to non-chunked, except we need * to swap out ntotal for the head-chunk-total. */ ntotal = s_cls->size; } if ((it->exptime != 0 && it->exptime < current_time) || item_is_flushed(it)) { /* Expired, don't save. */ save_item = 0; } else if (ch == NULL && (new_it = slab_rebalance_alloc(ntotal, slab_rebal.s_clsid)) == NULL) { /* Not a chunk of an item, and nomem. */ save_item = 0; slab_rebal.evictions_nomem++; } else if (ch != NULL && (new_it = slab_rebalance_alloc(s_cls->size, slab_rebal.s_clsid)) == NULL) { /* Is a chunk of an item, and nomem. */ save_item = 0; slab_rebal.evictions_nomem++; } else { /* Was whatever it was, and we have memory for it. */ save_item = 1; } pthread_mutex_unlock(&slabs_lock); if (save_item) { if (ch == NULL) { assert((new_it->it_flags & ITEM_CHUNKED) == 0); /* if free memory, memcpy. clear prev/next/h_bucket */ memcpy(new_it, it, ntotal); new_it->prev = 0; new_it->next = 0; new_it->h_next = 0; /* These are definitely required. else fails assert */ new_it->it_flags &= ~ITEM_LINKED; new_it->refcount = 0; do_item_replace(it, new_it, hv); /* Need to walk the chunks and repoint head */ if (new_it->it_flags & ITEM_CHUNKED) { item_chunk *fch = (item_chunk *) ITEM_schunk(new_it); fch->next->prev = fch; while (fch) { fch->head = new_it; fch = fch->next; } } it->refcount = 0; it->it_flags = ITEM_SLABBED|ITEM_FETCHED; #ifdef DEBUG_SLAB_MOVER memcpy(ITEM_key(it), "deadbeef", 8); #endif slab_rebal.rescues++; } else { item_chunk *nch = (item_chunk *) new_it; /* Chunks always have head chunk (the main it) */ ch->prev->next = nch; if (ch->next) ch->next->prev = nch; memcpy(nch, ch, ch->used + sizeof(item_chunk)); ch->refcount = 0; ch->it_flags = ITEM_SLABBED|ITEM_FETCHED; slab_rebal.chunk_rescues++; #ifdef DEBUG_SLAB_MOVER memcpy(ITEM_key((item *)ch), "deadbeef", 8); #endif refcount_decr(it); } slab_rebal.completed[offset] = 1; } else { /* unlink and mark as done if it's not * a chunked item as they require more book-keeping) */ STORAGE_delete(storage, it); if (!ch && (it->it_flags & ITEM_CHUNKED) == 0) { do_item_unlink(it, hv); it->it_flags = ITEM_SLABBED|ITEM_FETCHED; it->refcount = 0; #ifdef DEBUG_SLAB_MOVER memcpy(ITEM_key(it), "deadbeef", 8); #endif slab_rebal.completed[offset] = 1; } else { ntotal = ITEM_ntotal(it); do_item_unlink(it, hv); slabs_free(it, ntotal, slab_rebal.s_clsid); /* Swing around again later to remove it from the freelist. */ slab_rebal.busy_items++; was_busy++; } } item_trylock_unlock(hold_lock); pthread_mutex_lock(&slabs_lock); /* Always remove the ntotal, as we added it in during * do_slabs_alloc() when copying the item. */ break; case MOVE_FROM_SLAB: slab_rebal.completed[offset] = 1; it->refcount = 0; it->it_flags = ITEM_SLABBED|ITEM_FETCHED; #ifdef DEBUG_SLAB_MOVER memcpy(ITEM_key(it), "deadbeef", 8); #endif break; case MOVE_BUSY: case MOVE_LOCKED: slab_rebal.busy_items++; was_busy++; break; case MOVE_PASS: break; } pthread_mutex_unlock(&slabs_lock); } // Note: slab_rebal.* is occasionally protected under slabs_lock, but // the mover thread is the only user while active: so it's only necessary // for start/stop synchronization. slab_rebal.slab_pos = (char *)slab_rebal.slab_pos + s_cls->size; if (slab_rebal.slab_pos >= slab_rebal.slab_end) { /* Some items were busy, start again from the top */ if (slab_rebal.busy_items) { slab_rebal.slab_pos = slab_rebal.slab_start; STATS_LOCK(); stats.slab_reassign_busy_items += slab_rebal.busy_items; STATS_UNLOCK(); slab_rebal.busy_items = 0; slab_rebal.busy_loops++; } else { slab_rebal.done++; } } return was_busy; } static void slab_rebalance_finish(void) { slabclass_t *s_cls; slabclass_t *d_cls; int x; uint32_t rescues; uint32_t evictions_nomem; uint32_t inline_reclaim; uint32_t chunk_rescues; uint32_t busy_deletes; pthread_mutex_lock(&slabs_lock); s_cls = &slabclass[slab_rebal.s_clsid]; d_cls = &slabclass[slab_rebal.d_clsid]; #ifdef DEBUG_SLAB_MOVER /* If the algorithm is broken, live items can sneak in. */ slab_rebal.slab_pos = slab_rebal.slab_start; while (1) { item *it = slab_rebal.slab_pos; assert(it->it_flags == (ITEM_SLABBED|ITEM_FETCHED)); assert(memcmp(ITEM_key(it), "deadbeef", 8) == 0); it->it_flags = ITEM_SLABBED|ITEM_FETCHED; slab_rebal.slab_pos = (char *)slab_rebal.slab_pos + s_cls->size; if (slab_rebal.slab_pos >= slab_rebal.slab_end) break; } #endif /* At this point the stolen slab is completely clear. * We always kill the "first"/"oldest" slab page in the slab_list, so * shuffle the page list backwards and decrement. */ s_cls->slabs--; for (x = 0; x < s_cls->slabs; x++) { s_cls->slab_list[x] = s_cls->slab_list[x+1]; } d_cls->slab_list[d_cls->slabs++] = slab_rebal.slab_start; /* Don't need to split the page into chunks if we're just storing it */ if (slab_rebal.d_clsid > SLAB_GLOBAL_PAGE_POOL) { memset(slab_rebal.slab_start, 0, (size_t)settings.slab_page_size); split_slab_page_into_freelist(slab_rebal.slab_start, slab_rebal.d_clsid); } else if (slab_rebal.d_clsid == SLAB_GLOBAL_PAGE_POOL) { /* memset just enough to signal restart handler to skip */ memset(slab_rebal.slab_start, 0, sizeof(item)); /* mem_malloc'ed might be higher than mem_limit. */ mem_limit_reached = false; memory_release(); } slab_rebal.busy_loops = 0; slab_rebal.done = 0; slab_rebal.s_clsid = 0; slab_rebal.d_clsid = 0; slab_rebal.slab_start = NULL; slab_rebal.slab_end = NULL; slab_rebal.slab_pos = NULL; evictions_nomem = slab_rebal.evictions_nomem; inline_reclaim = slab_rebal.inline_reclaim; rescues = slab_rebal.rescues; chunk_rescues = slab_rebal.chunk_rescues; busy_deletes = slab_rebal.busy_deletes; slab_rebal.evictions_nomem = 0; slab_rebal.inline_reclaim = 0; slab_rebal.rescues = 0; slab_rebal.chunk_rescues = 0; slab_rebal.busy_deletes = 0; slab_rebalance_signal = 0; free(slab_rebal.completed); pthread_mutex_unlock(&slabs_lock); STATS_LOCK(); stats.slabs_moved++; stats.slab_reassign_rescues += rescues; stats.slab_reassign_evictions_nomem += evictions_nomem; stats.slab_reassign_inline_reclaim += inline_reclaim; stats.slab_reassign_chunk_rescues += chunk_rescues; stats.slab_reassign_busy_deletes += busy_deletes; stats_state.slab_reassign_running = false; STATS_UNLOCK(); if (settings.verbose > 1) { fprintf(stderr, "finished a slab move\n"); } } /* Slab mover thread. * Sits waiting for a condition to jump off and shovel some memory about */ static void *slab_rebalance_thread(void *arg) { int was_busy = 0; int backoff_timer = 1; int backoff_max = 1000; /* So we first pass into cond_wait with the mutex held */ mutex_lock(&slabs_rebalance_lock); /* Must finish moving page before stopping */ while (slab_rebalance_signal || do_run_slab_rebalance_thread) { if (slab_rebalance_signal == 1) { if (slab_rebalance_start() < 0) { /* Handle errors with more specificity as required. */ slab_rebalance_signal = 0; } was_busy = 0; } else if (slab_rebalance_signal && slab_rebal.slab_start != NULL) { was_busy = slab_rebalance_move(); } if (slab_rebal.done) { slab_rebalance_finish(); } else if (was_busy) { /* Stuck waiting for some items to unlock, so slow down a bit * to give them a chance to free up */ usleep(backoff_timer); backoff_timer = backoff_timer * 2; if (backoff_timer > backoff_max) backoff_timer = backoff_max; } if (slab_rebalance_signal == 0) { /* always hold this lock while we're running */ pthread_cond_wait(&slab_rebalance_cond, &slabs_rebalance_lock); } } // TODO: cancel in-flight slab page move mutex_unlock(&slabs_rebalance_lock); return NULL; } /* Iterate at most once through the slab classes and pick a "random" source. * I like this better than calling rand() since rand() is slow enough that we * can just check all of the classes once instead. */ static int slabs_reassign_pick_any(int dst) { static int cur = POWER_SMALLEST - 1; int tries = power_largest - POWER_SMALLEST + 1; for (; tries > 0; tries--) { cur++; if (cur > power_largest) cur = POWER_SMALLEST; if (cur == dst) continue; if (slabclass[cur].slabs > 1) { return cur; } } return -1; } static enum reassign_result_type do_slabs_reassign(int src, int dst) { bool nospare = false; if (slab_rebalance_signal != 0) return REASSIGN_RUNNING; if (src == dst) return REASSIGN_SRC_DST_SAME; /* Special indicator to choose ourselves. */ if (src == -1) { src = slabs_reassign_pick_any(dst); /* TODO: If we end up back at -1, return a new error type */ } if (src < SLAB_GLOBAL_PAGE_POOL || src > power_largest || dst < SLAB_GLOBAL_PAGE_POOL || dst > power_largest) return REASSIGN_BADCLASS; pthread_mutex_lock(&slabs_lock); if (slabclass[src].slabs < 2) nospare = true; pthread_mutex_unlock(&slabs_lock); if (nospare) return REASSIGN_NOSPARE; slab_rebal.s_clsid = src; slab_rebal.d_clsid = dst; slab_rebalance_signal = 1; pthread_cond_signal(&slab_rebalance_cond); return REASSIGN_OK; } enum reassign_result_type slabs_reassign(int src, int dst) { enum reassign_result_type ret; if (pthread_mutex_trylock(&slabs_rebalance_lock) != 0) { return REASSIGN_RUNNING; } ret = do_slabs_reassign(src, dst); pthread_mutex_unlock(&slabs_rebalance_lock); return ret; } /* If we hold this lock, rebalancer can't wake up or move */ void slabs_rebalancer_pause(void) { pthread_mutex_lock(&slabs_rebalance_lock); } void slabs_rebalancer_resume(void) { pthread_mutex_unlock(&slabs_rebalance_lock); } static pthread_t rebalance_tid; int start_slab_maintenance_thread(void) { int ret; slab_rebalance_signal = 0; slab_rebal.slab_start = NULL; if ((ret = pthread_create(&rebalance_tid, NULL, slab_rebalance_thread, NULL)) != 0) { fprintf(stderr, "Can't create rebal thread: %s\n", strerror(ret)); return -1; } return 0; } /* The maintenance thread is on a sleep/loop cycle, so it should join after a * short wait */ void stop_slab_maintenance_thread(void) { mutex_lock(&slabs_rebalance_lock); do_run_slab_rebalance_thread = 0; pthread_cond_signal(&slab_rebalance_cond); pthread_mutex_unlock(&slabs_rebalance_lock); /* Wait for the maintenance thread to stop */ pthread_join(rebalance_tid, NULL); } memcached-1.5.22/solaris_priv.c0000664000175000017500000000245713575022500013321 00000000000000#include #include #include #include "memcached.h" /* * this section of code will drop all (Solaris) privileges including * those normally granted to all userland process (basic privileges). The * effect of this is that after running this code, the process will not able * to fork(), exec(), etc. See privileges(5) for more information. */ void drop_privileges(void) { priv_set_t *privs = priv_str_to_set("basic", ",", NULL); if (privs == NULL) { perror("priv_str_to_set"); exit(EXIT_FAILURE); } (void)priv_delset(privs, PRIV_FILE_LINK_ANY); (void)priv_delset(privs, PRIV_PROC_EXEC); (void)priv_delset(privs, PRIV_PROC_FORK); (void)priv_delset(privs, PRIV_PROC_INFO); (void)priv_delset(privs, PRIV_PROC_SESSION); if (setppriv(PRIV_SET, PRIV_PERMITTED, privs) != 0) { perror("setppriv(PRIV_SET, PRIV_PERMITTED)"); exit(EXIT_FAILURE); } priv_emptyset(privs); if (setppriv(PRIV_SET, PRIV_INHERITABLE, privs) != 0) { perror("setppriv(PRIV_SET, PRIV_INHERITABLE)"); exit(EXIT_FAILURE); } if (setppriv(PRIV_SET, PRIV_LIMIT, privs) != 0) { perror("setppriv(PRIV_SET, PRIV_LIMIT)"); exit(EXIT_FAILURE); } priv_freeset(privs); } void setup_privilege_violations_handler(void) { // not needed } memcached-1.5.22/configure0000775000175000017500000073365413615454043012370 00000000000000#! /bin/sh # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.69 for memcached 1.5.22. # # Report bugs to . # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # # This configure script is free software; the Free Software Foundation # gives unlimited permission to copy, distribute and modify it. ## -------------------- ## ## M4sh Initialization. ## ## -------------------- ## # Be more Bourne compatible DUALCASE=1; export DUALCASE # for MKS sh if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which # is contrary to our usage. Disable this feature. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST else case `(set -o) 2>/dev/null` in #( *posix*) : set -o posix ;; #( *) : ;; esac fi as_nl=' ' export as_nl # Printing a long string crashes Solaris 7 /usr/bin/printf. as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo # Prefer a ksh shell builtin over an external printf program on Solaris, # but without wasting forks for bash or zsh. if test -z "$BASH_VERSION$ZSH_VERSION" \ && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then as_echo='print -r --' as_echo_n='print -rn --' elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then as_echo='printf %s\n' as_echo_n='printf %s' else if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' as_echo_n='/usr/ucb/echo -n' else as_echo_body='eval expr "X$1" : "X\\(.*\\)"' as_echo_n_body='eval arg=$1; case $arg in #( *"$as_nl"*) expr "X$arg" : "X\\(.*\\)$as_nl"; arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; esac; expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" ' export as_echo_n_body as_echo_n='sh -c $as_echo_n_body as_echo' fi export as_echo_body as_echo='sh -c $as_echo_body as_echo' fi # The user is always right. if test "${PATH_SEPARATOR+set}" != set; then PATH_SEPARATOR=: (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || PATH_SEPARATOR=';' } fi # IFS # We need space, tab and new line, in precisely that order. Quoting is # there to prevent editors from complaining about space-tab. # (If _AS_PATH_WALK were called with IFS unset, it would disable word # splitting by setting IFS to empty value.) IFS=" "" $as_nl" # Find who we are. Look in the path if we contain no directory separator. as_myself= case $0 in #(( *[\\/]* ) as_myself=$0 ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break done IFS=$as_save_IFS ;; esac # We did not find ourselves, most probably we were run as `sh COMMAND' # in which case we are not to be found in the path. if test "x$as_myself" = x; then as_myself=$0 fi if test ! -f "$as_myself"; then $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 exit 1 fi # Unset variables that we do not need and which cause bugs (e.g. in # pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" # suppresses any "Segmentation fault" message there. '((' could # trigger a bug in pdksh 5.2.14. for as_var in BASH_ENV ENV MAIL MAILPATH do eval test x\${$as_var+set} = xset \ && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : done PS1='$ ' PS2='> ' PS4='+ ' # NLS nuisances. LC_ALL=C export LC_ALL LANGUAGE=C export LANGUAGE # CDPATH. (unset CDPATH) >/dev/null 2>&1 && unset CDPATH # Use a proper internal environment variable to ensure we don't fall # into an infinite loop, continuously re-executing ourselves. if test x"${_as_can_reexec}" != xno && test "x$CONFIG_SHELL" != x; then _as_can_reexec=no; export _as_can_reexec; # We cannot yet assume a decent shell, so we have to provide a # neutralization value for shells without unset; and this also # works around shells that cannot unset nonexistent variables. # Preserve -v and -x to the replacement shell. BASH_ENV=/dev/null ENV=/dev/null (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV case $- in # (((( *v*x* | *x*v* ) as_opts=-vx ;; *v* ) as_opts=-v ;; *x* ) as_opts=-x ;; * ) as_opts= ;; esac exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # Admittedly, this is quite paranoid, since all the known shells bail # out after a failed `exec'. $as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2 as_fn_exit 255 fi # We don't want this to propagate to other subprocesses. { _as_can_reexec=; unset _as_can_reexec;} if test "x$CONFIG_SHELL" = x; then as_bourne_compatible="if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on \${1+\"\$@\"}, which # is contrary to our usage. Disable this feature. alias -g '\${1+\"\$@\"}'='\"\$@\"' setopt NO_GLOB_SUBST else case \`(set -o) 2>/dev/null\` in #( *posix*) : set -o posix ;; #( *) : ;; esac fi " as_required="as_fn_return () { (exit \$1); } as_fn_success () { as_fn_return 0; } as_fn_failure () { as_fn_return 1; } as_fn_ret_success () { return 0; } as_fn_ret_failure () { return 1; } exitcode=0 as_fn_success || { exitcode=1; echo as_fn_success failed.; } as_fn_failure && { exitcode=1; echo as_fn_failure succeeded.; } as_fn_ret_success || { exitcode=1; echo as_fn_ret_success failed.; } as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; } if ( set x; as_fn_ret_success y && test x = \"\$1\" ); then : else exitcode=1; echo positional parameters were not saved. fi test x\$exitcode = x0 || exit 1 test -x / || exit 1" as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" && test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1 test \$(( 1 + 1 )) = 2 || exit 1" if (eval "$as_required") 2>/dev/null; then : as_have_required=yes else as_have_required=no fi if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null; then : else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR as_found=false for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. as_found=: case $as_dir in #( /*) for as_base in sh bash ksh sh5; do # Try only shells that exist, to save several forks. as_shell=$as_dir/$as_base if { test -f "$as_shell" || test -f "$as_shell.exe"; } && { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$as_shell"; } 2>/dev/null; then : CONFIG_SHELL=$as_shell as_have_required=yes if { $as_echo "$as_bourne_compatible""$as_suggested" | as_run=a "$as_shell"; } 2>/dev/null; then : break 2 fi fi done;; esac as_found=false done $as_found || { if { test -f "$SHELL" || test -f "$SHELL.exe"; } && { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$SHELL"; } 2>/dev/null; then : CONFIG_SHELL=$SHELL as_have_required=yes fi; } IFS=$as_save_IFS if test "x$CONFIG_SHELL" != x; then : export CONFIG_SHELL # We cannot yet assume a decent shell, so we have to provide a # neutralization value for shells without unset; and this also # works around shells that cannot unset nonexistent variables. # Preserve -v and -x to the replacement shell. BASH_ENV=/dev/null ENV=/dev/null (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV case $- in # (((( *v*x* | *x*v* ) as_opts=-vx ;; *v* ) as_opts=-v ;; *x* ) as_opts=-x ;; * ) as_opts= ;; esac exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # Admittedly, this is quite paranoid, since all the known shells bail # out after a failed `exec'. $as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2 exit 255 fi if test x$as_have_required = xno; then : $as_echo "$0: This script requires a shell more modern than all" $as_echo "$0: the shells that I found on your system." if test x${ZSH_VERSION+set} = xset ; then $as_echo "$0: In particular, zsh $ZSH_VERSION has bugs and should" $as_echo "$0: be upgraded to zsh 4.3.4 or later." else $as_echo "$0: Please tell bug-autoconf@gnu.org and $0: memcached@googlegroups.com about your system, including $0: any error possibly output before this message. Then $0: install a modern shell, or manually run the script $0: under such a shell if you do have one." fi exit 1 fi fi fi SHELL=${CONFIG_SHELL-/bin/sh} export SHELL # Unset more variables known to interfere with behavior of common tools. CLICOLOR_FORCE= GREP_OPTIONS= unset CLICOLOR_FORCE GREP_OPTIONS ## --------------------- ## ## M4sh Shell Functions. ## ## --------------------- ## # as_fn_unset VAR # --------------- # Portably unset VAR. as_fn_unset () { { eval $1=; unset $1;} } as_unset=as_fn_unset # as_fn_set_status STATUS # ----------------------- # Set $? to STATUS, without forking. as_fn_set_status () { return $1 } # as_fn_set_status # as_fn_exit STATUS # ----------------- # Exit the shell with STATUS, even in a "trap 0" or "set -e" context. as_fn_exit () { set +e as_fn_set_status $1 exit $1 } # as_fn_exit # as_fn_mkdir_p # ------------- # Create "$as_dir" as a directory, including parents if necessary. as_fn_mkdir_p () { case $as_dir in #( -*) as_dir=./$as_dir;; esac test -d "$as_dir" || eval $as_mkdir_p || { as_dirs= while :; do case $as_dir in #( *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( *) as_qdir=$as_dir;; esac as_dirs="'$as_qdir' $as_dirs" as_dir=`$as_dirname -- "$as_dir" || $as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_dir" : 'X\(//\)[^/]' \| \ X"$as_dir" : 'X\(//\)$' \| \ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$as_dir" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" } # as_fn_mkdir_p # as_fn_executable_p FILE # ----------------------- # Test if FILE is an executable regular file. as_fn_executable_p () { test -f "$1" && test -x "$1" } # as_fn_executable_p # as_fn_append VAR VALUE # ---------------------- # Append the text in VALUE to the end of the definition contained in VAR. Take # advantage of any shell optimizations that allow amortized linear growth over # repeated appends, instead of the typical quadratic growth present in naive # implementations. if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : eval 'as_fn_append () { eval $1+=\$2 }' else as_fn_append () { eval $1=\$$1\$2 } fi # as_fn_append # as_fn_arith ARG... # ------------------ # Perform arithmetic evaluation on the ARGs, and store the result in the # global $as_val. Take advantage of shells that can avoid forks. The arguments # must be portable across $(()) and expr. if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : eval 'as_fn_arith () { as_val=$(( $* )) }' else as_fn_arith () { as_val=`expr "$@" || test $? -eq 1` } fi # as_fn_arith # as_fn_error STATUS ERROR [LINENO LOG_FD] # ---------------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the # script with STATUS, using 1 if that was 0. as_fn_error () { as_status=$1; test $as_status -eq 0 && as_status=1 if test "$4"; then as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi $as_echo "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error if expr a : '\(a\)' >/dev/null 2>&1 && test "X`expr 00001 : '.*\(...\)'`" = X001; then as_expr=expr else as_expr=false fi if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then as_basename=basename else as_basename=false fi if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then as_dirname=dirname else as_dirname=false fi as_me=`$as_basename -- "$0" || $as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ X"$0" : 'X\(//\)$' \| \ X"$0" : 'X\(/\)' \| . 2>/dev/null || $as_echo X/"$0" | sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/ q } /^X\/\(\/\/\)$/{ s//\1/ q } /^X\/\(\/\).*/{ s//\1/ q } s/.*/./; q'` # Avoid depending upon Character Ranges. as_cr_letters='abcdefghijklmnopqrstuvwxyz' as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' as_cr_Letters=$as_cr_letters$as_cr_LETTERS as_cr_digits='0123456789' as_cr_alnum=$as_cr_Letters$as_cr_digits as_lineno_1=$LINENO as_lineno_1a=$LINENO as_lineno_2=$LINENO as_lineno_2a=$LINENO eval 'test "x$as_lineno_1'$as_run'" != "x$as_lineno_2'$as_run'" && test "x`expr $as_lineno_1'$as_run' + 1`" = "x$as_lineno_2'$as_run'"' || { # Blame Lee E. McMahon (1931-1989) for sed's syntax. :-) sed -n ' p /[$]LINENO/= ' <$as_myself | sed ' s/[$]LINENO.*/&-/ t lineno b :lineno N :loop s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/ t loop s/-\n.*// ' >$as_me.lineno && chmod +x "$as_me.lineno" || { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; } # If we had to re-execute with $CONFIG_SHELL, we're ensured to have # already done that, so ensure we don't try to do so again and fall # in an infinite loop. This has already happened in practice. _as_can_reexec=no; export _as_can_reexec # Don't try to exec as it changes $[0], causing all sort of problems # (the dirname of $[0] is not the place where we might find the # original and so on. Autoconf is especially sensitive to this). . "./$as_me.lineno" # Exit status is that of the last command. exit } ECHO_C= ECHO_N= ECHO_T= case `echo -n x` in #((((( -n*) case `echo 'xy\c'` in *c*) ECHO_T=' ';; # ECHO_T is single tab character. xy) ECHO_C='\c';; *) echo `echo ksh88 bug on AIX 6.1` > /dev/null ECHO_T=' ';; esac;; *) ECHO_N='-n';; esac rm -f conf$$ conf$$.exe conf$$.file if test -d conf$$.dir; then rm -f conf$$.dir/conf$$.file else rm -f conf$$.dir mkdir conf$$.dir 2>/dev/null fi if (echo >conf$$.file) 2>/dev/null; then if ln -s conf$$.file conf$$ 2>/dev/null; then as_ln_s='ln -s' # ... but there are two gotchas: # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. # In both cases, we have to default to `cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then as_ln_s=ln else as_ln_s='cp -pR' fi else as_ln_s='cp -pR' fi rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file rmdir conf$$.dir 2>/dev/null if mkdir -p . 2>/dev/null; then as_mkdir_p='mkdir -p "$as_dir"' else test -d ./-p && rmdir ./-p as_mkdir_p=false fi as_test_x='test -x' as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" # Sed expression to map a string onto a valid variable name. as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" test -n "$DJDIR" || exec 7<&0 &1 # Name of the host. # hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status, # so uname gets run too. ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` # # Initializations. # ac_default_prefix=/usr/local ac_clean_files= ac_config_libobj_dir=. LIBOBJS= cross_compiling=no subdirs= MFLAGS= MAKEFLAGS= # Identity of this package. PACKAGE_NAME='memcached' PACKAGE_TARNAME='memcached' PACKAGE_VERSION='1.5.22' PACKAGE_STRING='memcached 1.5.22' PACKAGE_BUGREPORT='memcached@googlegroups.com' PACKAGE_URL='' ac_unique_file="memcached.c" # Factoring default headers for most tests. ac_includes_default="\ #include #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_STAT_H # include #endif #ifdef STDC_HEADERS # include # include #else # ifdef HAVE_STDLIB_H # include # endif #endif #ifdef HAVE_STRING_H # if !defined STDC_HEADERS && defined HAVE_MEMORY_H # include # endif # include #endif #ifdef HAVE_STRINGS_H # include #endif #ifdef HAVE_INTTYPES_H # include #endif #ifdef HAVE_STDINT_H # include #endif #ifdef HAVE_UNISTD_H # include #endif" ac_subst_vars='am__EXEEXT_FALSE am__EXEEXT_TRUE LTLIBOBJS LIBOBJS BUILD_SPECIFICATIONS_FALSE BUILD_SPECIFICATIONS_TRUE XSLTPROC XML2RFC BUILD_CACHE_FALSE BUILD_CACHE_TRUE BUILD_FREEBSD_PRIVS_FALSE BUILD_FREEBSD_PRIVS_TRUE BUILD_OPENBSD_PRIVS_FALSE BUILD_OPENBSD_PRIVS_TRUE BUILD_LINUX_PRIVS_FALSE BUILD_LINUX_PRIVS_TRUE BUILD_SOLARIS_PRIVS_FALSE BUILD_SOLARIS_PRIVS_TRUE PROFILER_FLAGS PROFILER PROFILER_LDFLAGS ENABLE_SASL DTRACEFLAGS ENABLE_ASAN_FALSE ENABLE_ASAN_TRUE ENABLE_TLS_FALSE ENABLE_TLS_TRUE ENABLE_ARM_CRC32_FALSE ENABLE_ARM_CRC32_TRUE ENABLE_EXTSTORE_FALSE ENABLE_EXTSTORE_TRUE ENABLE_SASL_FALSE ENABLE_SASL_TRUE DTRACE_INSTRUMENT_OBJ_FALSE DTRACE_INSTRUMENT_OBJ_TRUE BUILD_DTRACE_FALSE BUILD_DTRACE_TRUE DTRACE EGREP GREP CPP am__fastdepCC_FALSE am__fastdepCC_TRUE CCDEPMODE am__nodep AMDEPBACKSLASH AMDEP_FALSE AMDEP_TRUE am__quote am__include DEPDIR OBJEXT EXEEXT ac_ct_CC CPPFLAGS LDFLAGS CFLAGS CC AM_BACKSLASH AM_DEFAULT_VERBOSITY AM_DEFAULT_V AM_V am__untar am__tar AMTAR am__leading_dot SET_MAKE AWK mkdir_p MKDIR_P INSTALL_STRIP_PROGRAM STRIP install_sh MAKEINFO AUTOHEADER AUTOMAKE AUTOCONF ACLOCAL VERSION PACKAGE CYGPATH_W am__isrc INSTALL_DATA INSTALL_SCRIPT INSTALL_PROGRAM host_os host_vendor host_cpu host build_os build_vendor build_cpu build target_alias host_alias build_alias LIBS ECHO_T ECHO_N ECHO_C DEFS mandir localedir libdir psdir pdfdir dvidir htmldir infodir docdir oldincludedir includedir runstatedir localstatedir sharedstatedir sysconfdir datadir datarootdir libexecdir sbindir bindir program_transform_name prefix exec_prefix PACKAGE_URL PACKAGE_BUGREPORT PACKAGE_STRING PACKAGE_VERSION PACKAGE_TARNAME PACKAGE_NAME PATH_SEPARATOR SHELL' ac_subst_files='' ac_user_opts=' enable_option_checking enable_silent_rules enable_dependency_tracking enable_arm_crc32 enable_extstore enable_seccomp enable_sasl enable_sasl_pwdb enable_tls enable_asan enable_dtrace enable_coverage enable_64bit with_libevent with_libssl enable_docs ' ac_precious_vars='build_alias host_alias target_alias CC CFLAGS LDFLAGS LIBS CPPFLAGS CPP' # Initialize some variables set by options. ac_init_help= ac_init_version=false ac_unrecognized_opts= ac_unrecognized_sep= # The variables have the same names as the options, with # dashes changed to underlines. cache_file=/dev/null exec_prefix=NONE no_create= no_recursion= prefix=NONE program_prefix=NONE program_suffix=NONE program_transform_name=s,x,x, silent= site= srcdir= verbose= x_includes=NONE x_libraries=NONE # Installation directory options. # These are left unexpanded so users can "make install exec_prefix=/foo" # and all the variables that are supposed to be based on exec_prefix # by default will actually change. # Use braces instead of parens because sh, perl, etc. also accept them. # (The list follows the same order as the GNU Coding Standards.) bindir='${exec_prefix}/bin' sbindir='${exec_prefix}/sbin' libexecdir='${exec_prefix}/libexec' datarootdir='${prefix}/share' datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' infodir='${datarootdir}/info' htmldir='${docdir}' dvidir='${docdir}' pdfdir='${docdir}' psdir='${docdir}' libdir='${exec_prefix}/lib' localedir='${datarootdir}/locale' mandir='${datarootdir}/man' ac_prev= ac_dashdash= for ac_option do # If the previous option needs an argument, assign it. if test -n "$ac_prev"; then eval $ac_prev=\$ac_option ac_prev= continue fi case $ac_option in *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; *=) ac_optarg= ;; *) ac_optarg=yes ;; esac # Accept the important Cygnus configure options, so we can diagnose typos. case $ac_dashdash$ac_option in --) ac_dashdash=yes ;; -bindir | --bindir | --bindi | --bind | --bin | --bi) ac_prev=bindir ;; -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*) bindir=$ac_optarg ;; -build | --build | --buil | --bui | --bu) ac_prev=build_alias ;; -build=* | --build=* | --buil=* | --bui=* | --bu=*) build_alias=$ac_optarg ;; -cache-file | --cache-file | --cache-fil | --cache-fi \ | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) ac_prev=cache_file ;; -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*) cache_file=$ac_optarg ;; --config-cache | -C) cache_file=config.cache ;; -datadir | --datadir | --datadi | --datad) ac_prev=datadir ;; -datadir=* | --datadir=* | --datadi=* | --datad=*) datadir=$ac_optarg ;; -datarootdir | --datarootdir | --datarootdi | --datarootd | --dataroot \ | --dataroo | --dataro | --datar) ac_prev=datarootdir ;; -datarootdir=* | --datarootdir=* | --datarootdi=* | --datarootd=* \ | --dataroot=* | --dataroo=* | --dataro=* | --datar=*) datarootdir=$ac_optarg ;; -disable-* | --disable-*) ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "enable_$ac_useropt" "*) ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--disable-$ac_useropt_orig" ac_unrecognized_sep=', ';; esac eval enable_$ac_useropt=no ;; -docdir | --docdir | --docdi | --doc | --do) ac_prev=docdir ;; -docdir=* | --docdir=* | --docdi=* | --doc=* | --do=*) docdir=$ac_optarg ;; -dvidir | --dvidir | --dvidi | --dvid | --dvi | --dv) ac_prev=dvidir ;; -dvidir=* | --dvidir=* | --dvidi=* | --dvid=* | --dvi=* | --dv=*) dvidir=$ac_optarg ;; -enable-* | --enable-*) ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "enable_$ac_useropt" "*) ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig" ac_unrecognized_sep=', ';; esac eval enable_$ac_useropt=\$ac_optarg ;; -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \ | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \ | --exec | --exe | --ex) ac_prev=exec_prefix ;; -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \ | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \ | --exec=* | --exe=* | --ex=*) exec_prefix=$ac_optarg ;; -gas | --gas | --ga | --g) # Obsolete; use --with-gas. with_gas=yes ;; -help | --help | --hel | --he | -h) ac_init_help=long ;; -help=r* | --help=r* | --hel=r* | --he=r* | -hr*) ac_init_help=recursive ;; -help=s* | --help=s* | --hel=s* | --he=s* | -hs*) ac_init_help=short ;; -host | --host | --hos | --ho) ac_prev=host_alias ;; -host=* | --host=* | --hos=* | --ho=*) host_alias=$ac_optarg ;; -htmldir | --htmldir | --htmldi | --htmld | --html | --htm | --ht) ac_prev=htmldir ;; -htmldir=* | --htmldir=* | --htmldi=* | --htmld=* | --html=* | --htm=* \ | --ht=*) htmldir=$ac_optarg ;; -includedir | --includedir | --includedi | --included | --include \ | --includ | --inclu | --incl | --inc) ac_prev=includedir ;; -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \ | --includ=* | --inclu=* | --incl=* | --inc=*) includedir=$ac_optarg ;; -infodir | --infodir | --infodi | --infod | --info | --inf) ac_prev=infodir ;; -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*) infodir=$ac_optarg ;; -libdir | --libdir | --libdi | --libd) ac_prev=libdir ;; -libdir=* | --libdir=* | --libdi=* | --libd=*) libdir=$ac_optarg ;; -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \ | --libexe | --libex | --libe) ac_prev=libexecdir ;; -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \ | --libexe=* | --libex=* | --libe=*) libexecdir=$ac_optarg ;; -localedir | --localedir | --localedi | --localed | --locale) ac_prev=localedir ;; -localedir=* | --localedir=* | --localedi=* | --localed=* | --locale=*) localedir=$ac_optarg ;; -localstatedir | --localstatedir | --localstatedi | --localstated \ | --localstate | --localstat | --localsta | --localst | --locals) ac_prev=localstatedir ;; -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \ | --localstate=* | --localstat=* | --localsta=* | --localst=* | --locals=*) localstatedir=$ac_optarg ;; -mandir | --mandir | --mandi | --mand | --man | --ma | --m) ac_prev=mandir ;; -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*) mandir=$ac_optarg ;; -nfp | --nfp | --nf) # Obsolete; use --without-fp. with_fp=no ;; -no-create | --no-create | --no-creat | --no-crea | --no-cre \ | --no-cr | --no-c | -n) no_create=yes ;; -no-recursion | --no-recursion | --no-recursio | --no-recursi \ | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) no_recursion=yes ;; -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \ | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \ | --oldin | --oldi | --old | --ol | --o) ac_prev=oldincludedir ;; -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \ | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \ | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*) oldincludedir=$ac_optarg ;; -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) ac_prev=prefix ;; -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) prefix=$ac_optarg ;; -program-prefix | --program-prefix | --program-prefi | --program-pref \ | --program-pre | --program-pr | --program-p) ac_prev=program_prefix ;; -program-prefix=* | --program-prefix=* | --program-prefi=* \ | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*) program_prefix=$ac_optarg ;; -program-suffix | --program-suffix | --program-suffi | --program-suff \ | --program-suf | --program-su | --program-s) ac_prev=program_suffix ;; -program-suffix=* | --program-suffix=* | --program-suffi=* \ | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*) program_suffix=$ac_optarg ;; -program-transform-name | --program-transform-name \ | --program-transform-nam | --program-transform-na \ | --program-transform-n | --program-transform- \ | --program-transform | --program-transfor \ | --program-transfo | --program-transf \ | --program-trans | --program-tran \ | --progr-tra | --program-tr | --program-t) ac_prev=program_transform_name ;; -program-transform-name=* | --program-transform-name=* \ | --program-transform-nam=* | --program-transform-na=* \ | --program-transform-n=* | --program-transform-=* \ | --program-transform=* | --program-transfor=* \ | --program-transfo=* | --program-transf=* \ | --program-trans=* | --program-tran=* \ | --progr-tra=* | --program-tr=* | --program-t=*) program_transform_name=$ac_optarg ;; -pdfdir | --pdfdir | --pdfdi | --pdfd | --pdf | --pd) ac_prev=pdfdir ;; -pdfdir=* | --pdfdir=* | --pdfdi=* | --pdfd=* | --pdf=* | --pd=*) pdfdir=$ac_optarg ;; -psdir | --psdir | --psdi | --psd | --ps) ac_prev=psdir ;; -psdir=* | --psdir=* | --psdi=* | --psd=* | --ps=*) psdir=$ac_optarg ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ | -silent | --silent | --silen | --sile | --sil) silent=yes ;; -runstatedir | --runstatedir | --runstatedi | --runstated \ | --runstate | --runstat | --runsta | --runst | --runs \ | --run | --ru | --r) ac_prev=runstatedir ;; -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ | --run=* | --ru=* | --r=*) runstatedir=$ac_optarg ;; -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ | --sbi=* | --sb=*) sbindir=$ac_optarg ;; -sharedstatedir | --sharedstatedir | --sharedstatedi \ | --sharedstated | --sharedstate | --sharedstat | --sharedsta \ | --sharedst | --shareds | --shared | --share | --shar \ | --sha | --sh) ac_prev=sharedstatedir ;; -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \ | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \ | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \ | --sha=* | --sh=*) sharedstatedir=$ac_optarg ;; -site | --site | --sit) ac_prev=site ;; -site=* | --site=* | --sit=*) site=$ac_optarg ;; -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) ac_prev=srcdir ;; -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) srcdir=$ac_optarg ;; -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \ | --syscon | --sysco | --sysc | --sys | --sy) ac_prev=sysconfdir ;; -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \ | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*) sysconfdir=$ac_optarg ;; -target | --target | --targe | --targ | --tar | --ta | --t) ac_prev=target_alias ;; -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*) target_alias=$ac_optarg ;; -v | -verbose | --verbose | --verbos | --verbo | --verb) verbose=yes ;; -version | --version | --versio | --versi | --vers | -V) ac_init_version=: ;; -with-* | --with-*) ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "with_$ac_useropt" "*) ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig" ac_unrecognized_sep=', ';; esac eval with_$ac_useropt=\$ac_optarg ;; -without-* | --without-*) ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "with_$ac_useropt" "*) ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--without-$ac_useropt_orig" ac_unrecognized_sep=', ';; esac eval with_$ac_useropt=no ;; --x) # Obsolete; use --with-x. with_x=yes ;; -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \ | --x-incl | --x-inc | --x-in | --x-i) ac_prev=x_includes ;; -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \ | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*) x_includes=$ac_optarg ;; -x-libraries | --x-libraries | --x-librarie | --x-librari \ | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l) ac_prev=x_libraries ;; -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \ | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) x_libraries=$ac_optarg ;; -*) as_fn_error $? "unrecognized option: \`$ac_option' Try \`$0 --help' for more information" ;; *=*) ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='` # Reject names that are not valid shell variable names. case $ac_envvar in #( '' | [0-9]* | *[!_$as_cr_alnum]* ) as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; esac eval $ac_envvar=\$ac_optarg export $ac_envvar ;; *) # FIXME: should be removed in autoconf 3.0. $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2 expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2 : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}" ;; esac done if test -n "$ac_prev"; then ac_option=--`echo $ac_prev | sed 's/_/-/g'` as_fn_error $? "missing argument to $ac_option" fi if test -n "$ac_unrecognized_opts"; then case $enable_option_checking in no) ;; fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; esac fi # Check all directory arguments for consistency. for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. case $ac_val in */ ) ac_val=`expr "X$ac_val" : 'X\(.*[^/]\)' \| "X$ac_val" : 'X\(.*\)'` eval $ac_var=\$ac_val;; esac # Be sure to have absolute directory names. case $ac_val in [\\/$]* | ?:[\\/]* ) continue;; NONE | '' ) case $ac_var in *prefix ) continue;; esac;; esac as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" done # There might be people who depend on the old broken behavior: `$host' # used to hold the argument of --host etc. # FIXME: To remove some day. build=$build_alias host=$host_alias target=$target_alias # FIXME: To remove some day. if test "x$host_alias" != x; then if test "x$build_alias" = x; then cross_compiling=maybe elif test "x$build_alias" != "x$host_alias"; then cross_compiling=yes fi fi ac_tool_prefix= test -n "$host_alias" && ac_tool_prefix=$host_alias- test "$silent" = yes && exec 6>/dev/null ac_pwd=`pwd` && test -n "$ac_pwd" && ac_ls_di=`ls -di .` && ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || as_fn_error $? "working directory cannot be determined" test "X$ac_ls_di" = "X$ac_pwd_ls_di" || as_fn_error $? "pwd does not report name of working directory" # Find the source files, if location was not specified. if test -z "$srcdir"; then ac_srcdir_defaulted=yes # Try the directory containing this script, then the parent directory. ac_confdir=`$as_dirname -- "$as_myself" || $as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_myself" : 'X\(//\)[^/]' \| \ X"$as_myself" : 'X\(//\)$' \| \ X"$as_myself" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$as_myself" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` srcdir=$ac_confdir if test ! -r "$srcdir/$ac_unique_file"; then srcdir=.. fi else ac_srcdir_defaulted=no fi if test ! -r "$srcdir/$ac_unique_file"; then test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" fi ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" ac_abs_confdir=`( cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" pwd)` # When building in place, set srcdir=. if test "$ac_abs_confdir" = "$ac_pwd"; then srcdir=. fi # Remove unnecessary trailing slashes from srcdir. # Double slashes in file names in object file debugging info # mess up M-x gdb in Emacs. case $srcdir in */) srcdir=`expr "X$srcdir" : 'X\(.*[^/]\)' \| "X$srcdir" : 'X\(.*\)'`;; esac for ac_var in $ac_precious_vars; do eval ac_env_${ac_var}_set=\${${ac_var}+set} eval ac_env_${ac_var}_value=\$${ac_var} eval ac_cv_env_${ac_var}_set=\${${ac_var}+set} eval ac_cv_env_${ac_var}_value=\$${ac_var} done # # Report the --help message. # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF \`configure' configures memcached 1.5.22 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. Defaults for the options are specified in brackets. Configuration: -h, --help display this help and exit --help=short display options specific to this package --help=recursive display the short help of all the included packages -V, --version display version information and exit -q, --quiet, --silent do not print \`checking ...' messages --cache-file=FILE cache test results in FILE [disabled] -C, --config-cache alias for \`--cache-file=config.cache' -n, --no-create do not create output files --srcdir=DIR find the sources in DIR [configure dir or \`..'] Installation directories: --prefix=PREFIX install architecture-independent files in PREFIX [$ac_default_prefix] --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX [PREFIX] By default, \`make install' will install all the files in \`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify an installation prefix other than \`$ac_default_prefix' using \`--prefix', for instance \`--prefix=\$HOME'. For better control, use the options below. Fine tuning of the installation directories: --bindir=DIR user executables [EPREFIX/bin] --sbindir=DIR system admin executables [EPREFIX/sbin] --libexecdir=DIR program executables [EPREFIX/libexec] --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] --datadir=DIR read-only architecture-independent data [DATAROOTDIR] --infodir=DIR info documentation [DATAROOTDIR/info] --localedir=DIR locale-dependent data [DATAROOTDIR/locale] --mandir=DIR man documentation [DATAROOTDIR/man] --docdir=DIR documentation root [DATAROOTDIR/doc/memcached] --htmldir=DIR html documentation [DOCDIR] --dvidir=DIR dvi documentation [DOCDIR] --pdfdir=DIR pdf documentation [DOCDIR] --psdir=DIR ps documentation [DOCDIR] _ACEOF cat <<\_ACEOF Program names: --program-prefix=PREFIX prepend PREFIX to installed program names --program-suffix=SUFFIX append SUFFIX to installed program names --program-transform-name=PROGRAM run sed PROGRAM on installed program names System types: --build=BUILD configure for building on BUILD [guessed] --host=HOST cross-compile to build programs to run on HOST [BUILD] _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in short | recursive ) echo "Configuration of memcached 1.5.22:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --enable-silent-rules less verbose build output (undo: "make V=1") --disable-silent-rules verbose build output (undo: "make V=0") --enable-dependency-tracking do not reject slow dependency extractors --disable-dependency-tracking speeds up one-time build --enable-arm-crc32 Enable ARMv8 CRC32 instructions --enable-extstore Enable external storage EXPERIMENTAL --enable-seccomp Enable seccomp restrictions EXPERIMENTAL --enable-sasl Enable SASL authentication --enable-sasl-pwdb Enable plaintext password db --enable-tls Enable Transport Layer Security EXPERIMENTAL --enable-asan Compile with ASAN EXPERIMENTAL --enable-dtrace Enable dtrace probes --disable-coverage Disable code coverage --enable-64bit build 64bit version --disable-docs Disable documentation generation Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-libevent=PATH Specify path to libevent installation --with-libssl=PATH Specify path to libssl installation Some influential environment variables: CC C compiler command CFLAGS C compiler flags LDFLAGS linker flags, e.g. -L if you have libraries in a nonstandard directory LIBS libraries to pass to the linker, e.g. -l CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. Report bugs to . _ACEOF ac_status=$? fi if test "$ac_init_help" = "recursive"; then # If there are subdirs, report their specific --help. for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue test -d "$ac_dir" || { cd "$srcdir" && ac_pwd=`pwd` && srcdir=. && test -d "$ac_dir"; } || continue ac_builddir=. case "$ac_dir" in .) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` # A ".." for each directory in $ac_dir_suffix. ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` case $ac_top_builddir_sub in "") ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; esac ;; esac ac_abs_top_builddir=$ac_pwd ac_abs_builddir=$ac_pwd$ac_dir_suffix # for backward compatibility: ac_top_builddir=$ac_top_build_prefix case $srcdir in .) # We are building in place. ac_srcdir=. ac_top_srcdir=$ac_top_builddir_sub ac_abs_top_srcdir=$ac_pwd ;; [\\/]* | ?:[\\/]* ) # Absolute name. ac_srcdir=$srcdir$ac_dir_suffix; ac_top_srcdir=$srcdir ac_abs_top_srcdir=$srcdir ;; *) # Relative name. ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix ac_top_srcdir=$ac_top_build_prefix$srcdir ac_abs_top_srcdir=$ac_pwd/$srcdir ;; esac ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix cd "$ac_dir" || { ac_status=$?; continue; } # Check for guested configure. if test -f "$ac_srcdir/configure.gnu"; then echo && $SHELL "$ac_srcdir/configure.gnu" --help=recursive elif test -f "$ac_srcdir/configure"; then echo && $SHELL "$ac_srcdir/configure" --help=recursive else $as_echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2 fi || ac_status=$? cd "$ac_pwd" || { ac_status=$?; break; } done fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF memcached configure 1.5.22 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF exit fi ## ------------------------ ## ## Autoconf initialization. ## ## ------------------------ ## # ac_fn_c_try_compile LINENO # -------------------------- # Try to compile conftest.$ac_ext, and return whether this succeeded. ac_fn_c_try_compile () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack rm -f conftest.$ac_objext if { { ac_try="$ac_compile" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_compile") 2>conftest.err ac_status=$? if test -s conftest.err; then grep -v '^ *+' conftest.err >conftest.er1 cat conftest.er1 >&5 mv -f conftest.er1 conftest.err fi $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } && { test -z "$ac_c_werror_flag" || test ! -s conftest.err } && test -s conftest.$ac_objext; then : ac_retval=0 else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_compile # ac_fn_c_try_cpp LINENO # ---------------------- # Try to preprocess conftest.$ac_ext, and return whether this succeeded. ac_fn_c_try_cpp () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack if { { ac_try="$ac_cpp conftest.$ac_ext" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err ac_status=$? if test -s conftest.err; then grep -v '^ *+' conftest.err >conftest.er1 cat conftest.er1 >&5 mv -f conftest.er1 conftest.err fi $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } > conftest.i && { test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || test ! -s conftest.err }; then : ac_retval=0 else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_cpp # ac_fn_c_check_decl LINENO SYMBOL VAR INCLUDES # --------------------------------------------- # Tests whether SYMBOL is declared in INCLUDES, setting cache variable VAR # accordingly. ac_fn_c_check_decl () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack as_decl_name=`echo $2|sed 's/ *(.*//'` as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'` { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5 $as_echo_n "checking whether $as_decl_name is declared... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { #ifndef $as_decl_name #ifdef __cplusplus (void) $as_decl_use; #else (void) $as_decl_name; #endif #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : eval "$3=yes" else eval "$3=no" fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_decl # ac_fn_c_try_run LINENO # ---------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. Assumes # that executables *can* be run. ac_fn_c_try_run () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack if { { ac_try="$ac_link" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' { { case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_try") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; }; then : ac_retval=0 else $as_echo "$as_me: program exited with status $ac_status" >&5 $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=$ac_status fi rm -rf conftest.dSYM conftest_ipa8_conftest.oo eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_run # ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES # ------------------------------------------------------- # Tests whether HEADER exists and can be compiled using the include files in # INCLUDES, setting the cache variable VAR accordingly. ac_fn_c_check_header_compile () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 #include <$2> _ACEOF if ac_fn_c_try_compile "$LINENO"; then : eval "$3=yes" else eval "$3=no" fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_header_compile # ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES # ------------------------------------------------------- # Tests whether HEADER exists, giving a warning if it cannot be compiled using # the include files in INCLUDES and setting the cache variable VAR # accordingly. ac_fn_c_check_header_mongrel () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack if eval \${$3+:} false; then : { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } else # Is the header compilable? { $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 $as_echo_n "checking $2 usability... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 #include <$2> _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_header_compiler=yes else ac_header_compiler=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 $as_echo "$ac_header_compiler" >&6; } # Is the header present? { $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 $as_echo_n "checking $2 presence... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include <$2> _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : ac_header_preproc=yes else ac_header_preproc=no fi rm -f conftest.err conftest.i conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 $as_echo "$ac_header_preproc" >&6; } # So? What about this header? case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( yes:no: ) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 $as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 $as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} ;; no:yes:* ) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 $as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 $as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 $as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 $as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 $as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} ( $as_echo "## ----------------------------------------- ## ## Report this to memcached@googlegroups.com ## ## ----------------------------------------- ##" ) | sed "s/^/$as_me: WARNING: /" >&2 ;; esac { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else eval "$3=\$ac_header_compiler" fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_header_mongrel # ac_fn_c_try_link LINENO # ----------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. ac_fn_c_try_link () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack rm -f conftest.$ac_objext conftest$ac_exeext if { { ac_try="$ac_link" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link") 2>conftest.err ac_status=$? if test -s conftest.err; then grep -v '^ *+' conftest.err >conftest.er1 cat conftest.er1 >&5 mv -f conftest.er1 conftest.err fi $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } && { test -z "$ac_c_werror_flag" || test ! -s conftest.err } && test -s conftest$ac_exeext && { test "$cross_compiling" = yes || test -x conftest$ac_exeext }; then : ac_retval=0 else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 fi # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would # interfere with the next link command; also delete a directory that is # left behind by Apple's compiler. We do this before executing the actions. rm -rf conftest.dSYM conftest_ipa8_conftest.oo eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_link # ac_fn_c_compute_int LINENO EXPR VAR INCLUDES # -------------------------------------------- # Tries to find the compile-time value of EXPR in a program that includes # INCLUDES, setting VAR accordingly. Returns whether the value could be # computed ac_fn_c_compute_int () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack if test "$cross_compiling" = yes; then # Depending upon the size, compute the lo and hi bounds. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) >= 0)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_lo=0 ac_mid=0 while :; do cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) <= $ac_mid)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_hi=$ac_mid; break else as_fn_arith $ac_mid + 1 && ac_lo=$as_val if test $ac_lo -le $ac_mid; then ac_lo= ac_hi= break fi as_fn_arith 2 '*' $ac_mid + 1 && ac_mid=$as_val fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext done else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) < 0)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_hi=-1 ac_mid=-1 while :; do cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) >= $ac_mid)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_lo=$ac_mid; break else as_fn_arith '(' $ac_mid ')' - 1 && ac_hi=$as_val if test $ac_mid -le $ac_hi; then ac_lo= ac_hi= break fi as_fn_arith 2 '*' $ac_mid && ac_mid=$as_val fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext done else ac_lo= ac_hi= fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext # Binary search between lo and hi bounds. while test "x$ac_lo" != "x$ac_hi"; do as_fn_arith '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo && ac_mid=$as_val cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) <= $ac_mid)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_hi=$ac_mid else as_fn_arith '(' $ac_mid ')' + 1 && ac_lo=$as_val fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext done case $ac_lo in #(( ?*) eval "$3=\$ac_lo"; ac_retval=0 ;; '') ac_retval=1 ;; esac else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 static long int longval () { return $2; } static unsigned long int ulongval () { return $2; } #include #include int main () { FILE *f = fopen ("conftest.val", "w"); if (! f) return 1; if (($2) < 0) { long int i = longval (); if (i != ($2)) return 1; fprintf (f, "%ld", i); } else { unsigned long int i = ulongval (); if (i != ($2)) return 1; fprintf (f, "%lu", i); } /* Do not output a trailing newline, as this causes \r\n confusion on some platforms. */ return ferror (f) || fclose (f) != 0; ; return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : echo >>conftest.val; read $3 &5 $as_echo_n "checking for $2... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else eval "$3=no" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { if (sizeof ($2)) return 0; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { if (sizeof (($2))) return 0; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : else eval "$3=yes" fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_type # ac_fn_c_check_func LINENO FUNC VAR # ---------------------------------- # Tests whether FUNC exists, setting the cache variable VAR accordingly ac_fn_c_check_func () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Define $2 to an innocuous variant, in case declares $2. For example, HP-UX 11i declares gettimeofday. */ #define $2 innocuous_$2 /* System header to define __stub macros and hopefully few prototypes, which can conflict with char $2 (); below. Prefer to if __STDC__ is defined, since exists even on freestanding compilers. */ #ifdef __STDC__ # include #else # include #endif #undef $2 /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char $2 (); /* The GNU C library defines this for functions which it implements to always fail with ENOSYS. Some functions are actually named something starting with __ and the normal name is an alias. */ #if defined __stub_$2 || defined __stub___$2 choke me #endif int main () { return $2 (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : eval "$3=yes" else eval "$3=no" fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_func cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. It was created by memcached $as_me 1.5.22, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF exec 5>>config.log { cat <<_ASUNAME ## --------- ## ## Platform. ## ## --------- ## hostname = `(hostname || uname -n) 2>/dev/null | sed 1q` uname -m = `(uname -m) 2>/dev/null || echo unknown` uname -r = `(uname -r) 2>/dev/null || echo unknown` uname -s = `(uname -s) 2>/dev/null || echo unknown` uname -v = `(uname -v) 2>/dev/null || echo unknown` /usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown` /bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown` /bin/arch = `(/bin/arch) 2>/dev/null || echo unknown` /usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown` /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown` /usr/bin/hostinfo = `(/usr/bin/hostinfo) 2>/dev/null || echo unknown` /bin/machine = `(/bin/machine) 2>/dev/null || echo unknown` /usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown` /bin/universe = `(/bin/universe) 2>/dev/null || echo unknown` _ASUNAME as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. $as_echo "PATH: $as_dir" done IFS=$as_save_IFS } >&5 cat >&5 <<_ACEOF ## ----------- ## ## Core tests. ## ## ----------- ## _ACEOF # Keep a trace of the command line. # Strip out --no-create and --no-recursion so they do not pile up. # Strip out --silent because we don't want to record it for future runs. # Also quote any args containing shell meta-characters. # Make two passes to allow for proper duplicate-argument suppression. ac_configure_args= ac_configure_args0= ac_configure_args1= ac_must_keep_next=false for ac_pass in 1 2 do for ac_arg do case $ac_arg in -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ | -silent | --silent | --silen | --sile | --sil) continue ;; *\'*) ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; esac case $ac_pass in 1) as_fn_append ac_configure_args0 " '$ac_arg'" ;; 2) as_fn_append ac_configure_args1 " '$ac_arg'" if test $ac_must_keep_next = true; then ac_must_keep_next=false # Got value, back to normal. else case $ac_arg in *=* | --config-cache | -C | -disable-* | --disable-* \ | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \ | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \ | -with-* | --with-* | -without-* | --without-* | --x) case "$ac_configure_args0 " in "$ac_configure_args1"*" '$ac_arg' "* ) continue ;; esac ;; -* ) ac_must_keep_next=true ;; esac fi as_fn_append ac_configure_args " '$ac_arg'" ;; esac done done { ac_configure_args0=; unset ac_configure_args0;} { ac_configure_args1=; unset ac_configure_args1;} # When interrupted or exit'd, cleanup temporary files, and complete # config.log. We remove comments because anyway the quotes in there # would cause problems or look ugly. # WARNING: Use '\'' to represent an apostrophe within the trap. # WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug. trap 'exit_status=$? # Save into config.log some information that might help in debugging. { echo $as_echo "## ---------------- ## ## Cache variables. ## ## ---------------- ##" echo # The following way of writing the cache mishandles newlines in values, ( for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do eval ac_val=\$$ac_var case $ac_val in #( *${as_nl}*) case $ac_var in #( *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; esac case $ac_var in #( _ | IFS | as_nl) ;; #( BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( *) { eval $ac_var=; unset $ac_var;} ;; esac ;; esac done (set) 2>&1 | case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #( *${as_nl}ac_space=\ *) sed -n \ "s/'\''/'\''\\\\'\'''\''/g; s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p" ;; #( *) sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" ;; esac | sort ) echo $as_echo "## ----------------- ## ## Output variables. ## ## ----------------- ##" echo for ac_var in $ac_subst_vars do eval ac_val=\$$ac_var case $ac_val in *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; esac $as_echo "$ac_var='\''$ac_val'\''" done | sort echo if test -n "$ac_subst_files"; then $as_echo "## ------------------- ## ## File substitutions. ## ## ------------------- ##" echo for ac_var in $ac_subst_files do eval ac_val=\$$ac_var case $ac_val in *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; esac $as_echo "$ac_var='\''$ac_val'\''" done | sort echo fi if test -s confdefs.h; then $as_echo "## ----------- ## ## confdefs.h. ## ## ----------- ##" echo cat confdefs.h echo fi test "$ac_signal" != 0 && $as_echo "$as_me: caught signal $ac_signal" $as_echo "$as_me: exit $exit_status" } >&5 rm -f core *.core core.conftest.* && rm -f -r conftest* confdefs* conf$$* $ac_clean_files && exit $exit_status ' 0 for ac_signal in 1 2 13 15; do trap 'ac_signal='$ac_signal'; as_fn_exit 1' $ac_signal done ac_signal=0 # confdefs.h avoids OS command line length limits that DEFS can exceed. rm -f -r conftest* confdefs.h $as_echo "/* confdefs.h */" > confdefs.h # Predefined preprocessor variables. cat >>confdefs.h <<_ACEOF #define PACKAGE_NAME "$PACKAGE_NAME" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_TARNAME "$PACKAGE_TARNAME" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_VERSION "$PACKAGE_VERSION" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_STRING "$PACKAGE_STRING" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_URL "$PACKAGE_URL" _ACEOF # Let the site file select an alternate cache file if it wants to. # Prefer an explicitly selected file to automatically selected ones. ac_site_file1=NONE ac_site_file2=NONE if test -n "$CONFIG_SITE"; then # We do not want a PATH search for config.site. case $CONFIG_SITE in #(( -*) ac_site_file1=./$CONFIG_SITE;; */*) ac_site_file1=$CONFIG_SITE;; *) ac_site_file1=./$CONFIG_SITE;; esac elif test "x$prefix" != xNONE; then ac_site_file1=$prefix/share/config.site ac_site_file2=$prefix/etc/config.site else ac_site_file1=$ac_default_prefix/share/config.site ac_site_file2=$ac_default_prefix/etc/config.site fi for ac_site_file in "$ac_site_file1" "$ac_site_file2" do test "x$ac_site_file" = xNONE && continue if test /dev/null != "$ac_site_file" && test -r "$ac_site_file"; then { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 $as_echo "$as_me: loading site script $ac_site_file" >&6;} sed 's/^/| /' "$ac_site_file" >&5 . "$ac_site_file" \ || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "failed to load site script $ac_site_file See \`config.log' for more details" "$LINENO" 5; } fi done if test -r "$cache_file"; then # Some versions of bash will fail to source /dev/null (special files # actually), so we avoid doing that. DJGPP emulates it as a regular file. if test /dev/null != "$cache_file" && test -f "$cache_file"; then { $as_echo "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5 $as_echo "$as_me: loading cache $cache_file" >&6;} case $cache_file in [\\/]* | ?:[\\/]* ) . "$cache_file";; *) . "./$cache_file";; esac fi else { $as_echo "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5 $as_echo "$as_me: creating cache $cache_file" >&6;} >$cache_file fi # Check that the precious variables saved in the cache have kept the same # value. ac_cache_corrupted=false for ac_var in $ac_precious_vars; do eval ac_old_set=\$ac_cv_env_${ac_var}_set eval ac_new_set=\$ac_env_${ac_var}_set eval ac_old_val=\$ac_cv_env_${ac_var}_value eval ac_new_val=\$ac_env_${ac_var}_value case $ac_old_set,$ac_new_set in set,) { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 $as_echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} ac_cache_corrupted=: ;; ,set) { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5 $as_echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} ac_cache_corrupted=: ;; ,);; *) if test "x$ac_old_val" != "x$ac_new_val"; then # differences in whitespace do not lead to failure. ac_old_val_w=`echo x $ac_old_val` ac_new_val_w=`echo x $ac_new_val` if test "$ac_old_val_w" != "$ac_new_val_w"; then { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5 $as_echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} ac_cache_corrupted=: else { $as_echo "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5 $as_echo "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;} eval $ac_var=\$ac_old_val fi { $as_echo "$as_me:${as_lineno-$LINENO}: former value: \`$ac_old_val'" >&5 $as_echo "$as_me: former value: \`$ac_old_val'" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: current value: \`$ac_new_val'" >&5 $as_echo "$as_me: current value: \`$ac_new_val'" >&2;} fi;; esac # Pass precious variables to config.status. if test "$ac_new_set" = set; then case $ac_new_val in *\'*) ac_arg=$ac_var=`$as_echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; *) ac_arg=$ac_var=$ac_new_val ;; esac case " $ac_configure_args " in *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy. *) as_fn_append ac_configure_args " '$ac_arg'" ;; esac fi done if $ac_cache_corrupted; then { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 $as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 fi ## -------------------- ## ## Main body of script. ## ## -------------------- ## ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu ac_aux_dir= for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do if test -f "$ac_dir/install-sh"; then ac_aux_dir=$ac_dir ac_install_sh="$ac_aux_dir/install-sh -c" break elif test -f "$ac_dir/install.sh"; then ac_aux_dir=$ac_dir ac_install_sh="$ac_aux_dir/install.sh -c" break elif test -f "$ac_dir/shtool"; then ac_aux_dir=$ac_dir ac_install_sh="$ac_aux_dir/shtool install -c" break fi done if test -z "$ac_aux_dir"; then as_fn_error $? "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 fi # These three variables are undocumented and unsupported, # and are intended to be withdrawn in a future Autoconf release. # They can cause serious problems if a builder's source tree is in a directory # whose full name contains unusual characters. ac_config_guess="$SHELL $ac_aux_dir/config.guess" # Please don't use this var. ac_config_sub="$SHELL $ac_aux_dir/config.sub" # Please don't use this var. ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. # Make sure we can run config.sub. $SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 || as_fn_error $? "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5 { $as_echo "$as_me:${as_lineno-$LINENO}: checking build system type" >&5 $as_echo_n "checking build system type... " >&6; } if ${ac_cv_build+:} false; then : $as_echo_n "(cached) " >&6 else ac_build_alias=$build_alias test "x$ac_build_alias" = x && ac_build_alias=`$SHELL "$ac_aux_dir/config.guess"` test "x$ac_build_alias" = x && as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5 ac_cv_build=`$SHELL "$ac_aux_dir/config.sub" $ac_build_alias` || as_fn_error $? "$SHELL $ac_aux_dir/config.sub $ac_build_alias failed" "$LINENO" 5 fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5 $as_echo "$ac_cv_build" >&6; } case $ac_cv_build in *-*-*) ;; *) as_fn_error $? "invalid value of canonical build" "$LINENO" 5;; esac build=$ac_cv_build ac_save_IFS=$IFS; IFS='-' set x $ac_cv_build shift build_cpu=$1 build_vendor=$2 shift; shift # Remember, the first character of IFS is used to create $*, # except with old shells: build_os=$* IFS=$ac_save_IFS case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac { $as_echo "$as_me:${as_lineno-$LINENO}: checking host system type" >&5 $as_echo_n "checking host system type... " >&6; } if ${ac_cv_host+:} false; then : $as_echo_n "(cached) " >&6 else if test "x$host_alias" = x; then ac_cv_host=$ac_cv_build else ac_cv_host=`$SHELL "$ac_aux_dir/config.sub" $host_alias` || as_fn_error $? "$SHELL $ac_aux_dir/config.sub $host_alias failed" "$LINENO" 5 fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_host" >&5 $as_echo "$ac_cv_host" >&6; } case $ac_cv_host in *-*-*) ;; *) as_fn_error $? "invalid value of canonical host" "$LINENO" 5;; esac host=$ac_cv_host ac_save_IFS=$IFS; IFS='-' set x $ac_cv_host shift host_cpu=$1 host_vendor=$2 shift; shift # Remember, the first character of IFS is used to create $*, # except with old shells: host_os=$* IFS=$ac_save_IFS case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac am__api_version='1.15' # Find a good install program. We prefer a C program (faster), # so one script is as good as another. But avoid the broken or # incompatible versions: # SysV /etc/install, /usr/sbin/install # SunOS /usr/etc/install # IRIX /sbin/install # AIX /bin/install # AmigaOS /C/install, which installs bootblocks on floppy discs # AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag # AFS /usr/afsws/bin/install, which mishandles nonexistent args # SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" # OS/2's system install, which has a completely different semantic # ./install, which can be erroneously created by make from ./install.sh. # Reject install programs that cannot install multiple files. { $as_echo "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5 $as_echo_n "checking for a BSD-compatible install... " >&6; } if test -z "$INSTALL"; then if ${ac_cv_path_install+:} false; then : $as_echo_n "(cached) " >&6 else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. # Account for people who put trailing slashes in PATH elements. case $as_dir/ in #(( ./ | .// | /[cC]/* | \ /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \ ?:[\\/]os2[\\/]install[\\/]* | ?:[\\/]OS2[\\/]INSTALL[\\/]* | \ /usr/ucb/* ) ;; *) # OSF1 and SCO ODT 3.0 have their own names for install. # Don't use installbsd from OSF since it installs stuff as root # by default. for ac_prog in ginstall scoinst install; do for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then if test $ac_prog = install && grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then # AIX install. It has an incompatible calling convention. : elif test $ac_prog = install && grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then # program-specific install script used by HP pwplus--don't use. : else rm -rf conftest.one conftest.two conftest.dir echo one > conftest.one echo two > conftest.two mkdir conftest.dir if "$as_dir/$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir" && test -s conftest.one && test -s conftest.two && test -s conftest.dir/conftest.one && test -s conftest.dir/conftest.two then ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c" break 3 fi fi fi done done ;; esac done IFS=$as_save_IFS rm -rf conftest.one conftest.two conftest.dir fi if test "${ac_cv_path_install+set}" = set; then INSTALL=$ac_cv_path_install else # As a last resort, use the slow shell script. Don't cache a # value for INSTALL within a source directory, because that will # break other packages using the cache if that directory is # removed, or if the value is a relative name. INSTALL=$ac_install_sh fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $INSTALL" >&5 $as_echo "$INSTALL" >&6; } # Use test -z because SunOS4 sh mishandles braces in ${var-val}. # It thinks the first close brace ends the variable substitution. test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}' test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether build environment is sane" >&5 $as_echo_n "checking whether build environment is sane... " >&6; } # Reject unsafe characters in $srcdir or the absolute working directory # name. Accept space and tab only in the latter. am_lf=' ' case `pwd` in *[\\\"\#\$\&\'\`$am_lf]*) as_fn_error $? "unsafe absolute working directory name" "$LINENO" 5;; esac case $srcdir in *[\\\"\#\$\&\'\`$am_lf\ \ ]*) as_fn_error $? "unsafe srcdir value: '$srcdir'" "$LINENO" 5;; esac # Do 'set' in a subshell so we don't clobber the current shell's # arguments. Must try -L first in case configure is actually a # symlink; some systems play weird games with the mod time of symlinks # (eg FreeBSD returns the mod time of the symlink's containing # directory). if ( am_has_slept=no for am_try in 1 2; do echo "timestamp, slept: $am_has_slept" > conftest.file set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null` if test "$*" = "X"; then # -L didn't work. set X `ls -t "$srcdir/configure" conftest.file` fi if test "$*" != "X $srcdir/configure conftest.file" \ && test "$*" != "X conftest.file $srcdir/configure"; then # If neither matched, then we have a broken ls. This can happen # if, for instance, CONFIG_SHELL is bash and it inherits a # broken ls alias from the environment. This has actually # happened. Such a system could not be considered "sane". as_fn_error $? "ls -t appears to fail. Make sure there is not a broken alias in your environment" "$LINENO" 5 fi if test "$2" = conftest.file || test $am_try -eq 2; then break fi # Just in case. sleep 1 am_has_slept=yes done test "$2" = conftest.file ) then # Ok. : else as_fn_error $? "newly created file is older than distributed files! Check your system clock" "$LINENO" 5 fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } # If we didn't sleep, we still need to ensure time stamps of config.status and # generated files are strictly newer. am_sleep_pid= if grep 'slept: no' conftest.file >/dev/null 2>&1; then ( sleep 1 ) & am_sleep_pid=$! fi rm -f conftest.file test "$program_prefix" != NONE && program_transform_name="s&^&$program_prefix&;$program_transform_name" # Use a double $ so make ignores it. test "$program_suffix" != NONE && program_transform_name="s&\$&$program_suffix&;$program_transform_name" # Double any \ or $. # By default was `s,x,x', remove it if useless. ac_script='s/[\\$]/&&/g;s/;s,x,x,$//' program_transform_name=`$as_echo "$program_transform_name" | sed "$ac_script"` # Expand $ac_aux_dir to an absolute path. am_aux_dir=`cd "$ac_aux_dir" && pwd` if test x"${MISSING+set}" != xset; then case $am_aux_dir in *\ * | *\ *) MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;; *) MISSING="\${SHELL} $am_aux_dir/missing" ;; esac fi # Use eval to expand $SHELL if eval "$MISSING --is-lightweight"; then am_missing_run="$MISSING " else am_missing_run= { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: 'missing' script is too old or missing" >&5 $as_echo "$as_me: WARNING: 'missing' script is too old or missing" >&2;} fi if test x"${install_sh+set}" != xset; then case $am_aux_dir in *\ * | *\ *) install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;; *) install_sh="\${SHELL} $am_aux_dir/install-sh" esac fi # Installed binaries are usually stripped using 'strip' when the user # run "make install-strip". However 'strip' might not be the right # tool to use in cross-compilation environments, therefore Automake # will honor the 'STRIP' environment variable to overrule this program. if test "$cross_compiling" != no; then if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args. set dummy ${ac_tool_prefix}strip; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_STRIP+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$STRIP"; then ac_cv_prog_STRIP="$STRIP" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_STRIP="${ac_tool_prefix}strip" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi STRIP=$ac_cv_prog_STRIP if test -n "$STRIP"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STRIP" >&5 $as_echo "$STRIP" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi fi if test -z "$ac_cv_prog_STRIP"; then ac_ct_STRIP=$STRIP # Extract the first word of "strip", so it can be a program name with args. set dummy strip; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_ac_ct_STRIP+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_STRIP"; then ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_STRIP="strip" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP if test -n "$ac_ct_STRIP"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_STRIP" >&5 $as_echo "$ac_ct_STRIP" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi if test "x$ac_ct_STRIP" = x; then STRIP=":" else case $cross_compiling:$ac_tool_warned in yes:) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 $as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} ac_tool_warned=yes ;; esac STRIP=$ac_ct_STRIP fi else STRIP="$ac_cv_prog_STRIP" fi fi INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s" { $as_echo "$as_me:${as_lineno-$LINENO}: checking for a thread-safe mkdir -p" >&5 $as_echo_n "checking for a thread-safe mkdir -p... " >&6; } if test -z "$MKDIR_P"; then if ${ac_cv_path_mkdir+:} false; then : $as_echo_n "(cached) " >&6 else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH$PATH_SEPARATOR/opt/sfw/bin do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_prog in mkdir gmkdir; do for ac_exec_ext in '' $ac_executable_extensions; do as_fn_executable_p "$as_dir/$ac_prog$ac_exec_ext" || continue case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #( 'mkdir (GNU coreutils) '* | \ 'mkdir (coreutils) '* | \ 'mkdir (fileutils) '4.1*) ac_cv_path_mkdir=$as_dir/$ac_prog$ac_exec_ext break 3;; esac done done done IFS=$as_save_IFS fi test -d ./--version && rmdir ./--version if test "${ac_cv_path_mkdir+set}" = set; then MKDIR_P="$ac_cv_path_mkdir -p" else # As a last resort, use the slow shell script. Don't cache a # value for MKDIR_P within a source directory, because that will # break other packages using the cache if that directory is # removed, or if the value is a relative name. MKDIR_P="$ac_install_sh -d" fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MKDIR_P" >&5 $as_echo "$MKDIR_P" >&6; } for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_AWK+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$AWK"; then ac_cv_prog_AWK="$AWK" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_AWK="$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi AWK=$ac_cv_prog_AWK if test -n "$AWK"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5 $as_echo "$AWK" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi test -n "$AWK" && break done { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${MAKE-make} sets \$(MAKE)" >&5 $as_echo_n "checking whether ${MAKE-make} sets \$(MAKE)... " >&6; } set x ${MAKE-make} ac_make=`$as_echo "$2" | sed 's/+/p/g; s/[^a-zA-Z0-9_]/_/g'` if eval \${ac_cv_prog_make_${ac_make}_set+:} false; then : $as_echo_n "(cached) " >&6 else cat >conftest.make <<\_ACEOF SHELL = /bin/sh all: @echo '@@@%%%=$(MAKE)=@@@%%%' _ACEOF # GNU make sometimes prints "make[1]: Entering ...", which would confuse us. case `${MAKE-make} -f conftest.make 2>/dev/null` in *@@@%%%=?*=@@@%%%*) eval ac_cv_prog_make_${ac_make}_set=yes;; *) eval ac_cv_prog_make_${ac_make}_set=no;; esac rm -f conftest.make fi if eval test \$ac_cv_prog_make_${ac_make}_set = yes; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } SET_MAKE= else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } SET_MAKE="MAKE=${MAKE-make}" fi rm -rf .tst 2>/dev/null mkdir .tst 2>/dev/null if test -d .tst; then am__leading_dot=. else am__leading_dot=_ fi rmdir .tst 2>/dev/null # Check whether --enable-silent-rules was given. if test "${enable_silent_rules+set}" = set; then : enableval=$enable_silent_rules; fi case $enable_silent_rules in # ((( yes) AM_DEFAULT_VERBOSITY=0;; no) AM_DEFAULT_VERBOSITY=1;; *) AM_DEFAULT_VERBOSITY=1;; esac am_make=${MAKE-make} { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $am_make supports nested variables" >&5 $as_echo_n "checking whether $am_make supports nested variables... " >&6; } if ${am_cv_make_support_nested_variables+:} false; then : $as_echo_n "(cached) " >&6 else if $as_echo 'TRUE=$(BAR$(V)) BAR0=false BAR1=true V=1 am__doit: @$(TRUE) .PHONY: am__doit' | $am_make -f - >/dev/null 2>&1; then am_cv_make_support_nested_variables=yes else am_cv_make_support_nested_variables=no fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_make_support_nested_variables" >&5 $as_echo "$am_cv_make_support_nested_variables" >&6; } if test $am_cv_make_support_nested_variables = yes; then AM_V='$(V)' AM_DEFAULT_V='$(AM_DEFAULT_VERBOSITY)' else AM_V=$AM_DEFAULT_VERBOSITY AM_DEFAULT_V=$AM_DEFAULT_VERBOSITY fi AM_BACKSLASH='\' if test "`cd $srcdir && pwd`" != "`pwd`"; then # Use -I$(srcdir) only when $(srcdir) != ., so that make's output # is not polluted with repeated "-I." am__isrc=' -I$(srcdir)' # test to see if srcdir already configured if test -f $srcdir/config.status; then as_fn_error $? "source directory already configured; run \"make distclean\" there first" "$LINENO" 5 fi fi # test whether we have cygpath if test -z "$CYGPATH_W"; then if (cygpath --version) >/dev/null 2>/dev/null; then CYGPATH_W='cygpath -w' else CYGPATH_W=echo fi fi # Define the identity of the package. PACKAGE='memcached' VERSION='1.5.22' cat >>confdefs.h <<_ACEOF #define PACKAGE "$PACKAGE" _ACEOF cat >>confdefs.h <<_ACEOF #define VERSION "$VERSION" _ACEOF # Some tools Automake needs. ACLOCAL=${ACLOCAL-"${am_missing_run}aclocal-${am__api_version}"} AUTOCONF=${AUTOCONF-"${am_missing_run}autoconf"} AUTOMAKE=${AUTOMAKE-"${am_missing_run}automake-${am__api_version}"} AUTOHEADER=${AUTOHEADER-"${am_missing_run}autoheader"} MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"} # For better backward compatibility. To be removed once Automake 1.9.x # dies out for good. For more background, see: # # mkdir_p='$(MKDIR_P)' # We need awk for the "check" target (and possibly the TAP driver). The # system "awk" is bad on some platforms. # Always define AMTAR for backward compatibility. Yes, it's still used # in the wild :-( We should find a proper way to deprecate it ... AMTAR='$${TAR-tar}' # We'll loop over all known methods to create a tar archive until one works. _am_tools='gnutar pax cpio none' am__tar='$${TAR-tar} chof - "$$tardir"' am__untar='$${TAR-tar} xf -' # POSIX will say in a future version that running "rm -f" with no argument # is OK; and we want to be able to make that assumption in our Makefile # recipes. So use an aggressive probe to check that the usage we want is # actually supported "in the wild" to an acceptable degree. # See automake bug#10828. # To make any issue more visible, cause the running configure to be aborted # by default if the 'rm' program in use doesn't match our expectations; the # user can still override this though. if rm -f && rm -fr && rm -rf; then : OK; else cat >&2 <<'END' Oops! Your 'rm' program seems unable to run without file operands specified on the command line, even when the '-f' option is present. This is contrary to the behaviour of most rm programs out there, and not conforming with the upcoming POSIX standard: Please tell bug-automake@gnu.org about your system, including the value of your $PATH and any error possibly output before this message. This can help us improve future automake versions. END if test x"$ACCEPT_INFERIOR_RM_PROGRAM" = x"yes"; then echo 'Configuration will proceed anyway, since you have set the' >&2 echo 'ACCEPT_INFERIOR_RM_PROGRAM variable to "yes"' >&2 echo >&2 else cat >&2 <<'END' Aborting the configuration process, to ensure you take notice of the issue. You can download and install GNU coreutils to get an 'rm' implementation that behaves properly: . If you want to complete the configuration process using your problematic 'rm' anyway, export the environment variable ACCEPT_INFERIOR_RM_PROGRAM to "yes", and re-run configure. END as_fn_error $? "Your 'rm' program is bad, sorry." "$LINENO" 5 fi fi ac_config_headers="$ac_config_headers config.h" ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args. set dummy ${ac_tool_prefix}gcc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_CC="${ac_tool_prefix}gcc" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi CC=$ac_cv_prog_CC if test -n "$CC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 $as_echo "$CC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi fi if test -z "$ac_cv_prog_CC"; then ac_ct_CC=$CC # Extract the first word of "gcc", so it can be a program name with args. set dummy gcc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_ac_ct_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_CC="gcc" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 $as_echo "$ac_ct_CC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi if test "x$ac_ct_CC" = x; then CC="" else case $cross_compiling:$ac_tool_warned in yes:) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 $as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} ac_tool_warned=yes ;; esac CC=$ac_ct_CC fi else CC="$ac_cv_prog_CC" fi if test -z "$CC"; then if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. set dummy ${ac_tool_prefix}cc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_CC="${ac_tool_prefix}cc" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi CC=$ac_cv_prog_CC if test -n "$CC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 $as_echo "$CC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi fi fi if test -z "$CC"; then # Extract the first word of "cc", so it can be a program name with args. set dummy cc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else ac_prog_rejected=no as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then ac_prog_rejected=yes continue fi ac_cv_prog_CC="cc" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS if test $ac_prog_rejected = yes; then # We found a bogon in the path, so make sure we never use it. set dummy $ac_cv_prog_CC shift if test $# != 0; then # We chose a different compiler from the bogus one. # However, it has the same basename, so the bogon will be chosen # first if we set CC to just the basename; use the full file name. shift ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@" fi fi fi fi CC=$ac_cv_prog_CC if test -n "$CC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 $as_echo "$CC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi fi if test -z "$CC"; then if test -n "$ac_tool_prefix"; then for ac_prog in cl.exe do # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. set dummy $ac_tool_prefix$ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_CC="$ac_tool_prefix$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi CC=$ac_cv_prog_CC if test -n "$CC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 $as_echo "$CC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi test -n "$CC" && break done fi if test -z "$CC"; then ac_ct_CC=$CC for ac_prog in cl.exe do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_ac_ct_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_CC="$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 $as_echo "$ac_ct_CC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi test -n "$ac_ct_CC" && break done if test "x$ac_ct_CC" = x; then CC="" else case $cross_compiling:$ac_tool_warned in yes:) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 $as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} ac_tool_warned=yes ;; esac CC=$ac_ct_CC fi fi fi test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "no acceptable C compiler found in \$PATH See \`config.log' for more details" "$LINENO" 5; } # Provide some information about the compiler. $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 set X $ac_compile ac_compiler=$2 for ac_option in --version -v -V -qversion; do { { ac_try="$ac_compiler $ac_option >&5" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_compiler $ac_option >&5") 2>conftest.err ac_status=$? if test -s conftest.err; then sed '10a\ ... rest of stderr output deleted ... 10q' conftest.err >conftest.er1 cat conftest.er1 >&5 fi rm -f conftest.er1 conftest.err $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } done cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF ac_clean_files_save=$ac_clean_files ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out" # Try to create an executable without -o first, disregard a.out. # It will help us diagnose broken compilers, and finding out an intuition # of exeext. { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5 $as_echo_n "checking whether the C compiler works... " >&6; } ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'` # The possible output files: ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*" ac_rmfiles= for ac_file in $ac_files do case $ac_file in *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; * ) ac_rmfiles="$ac_rmfiles $ac_file";; esac done rm -f $ac_rmfiles if { { ac_try="$ac_link_default" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link_default") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then : # Autoconf-2.13 could set the ac_cv_exeext variable to `no'. # So ignore a value of `no', otherwise this would lead to `EXEEXT = no' # in a Makefile. We should not override ac_cv_exeext if it was cached, # so that the user can short-circuit this test for compilers unknown to # Autoconf. for ac_file in $ac_files '' do test -f "$ac_file" || continue case $ac_file in *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; [ab].out ) # We found the default executable, but exeext='' is most # certainly right. break;; *.* ) if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no; then :; else ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` fi # We set ac_cv_exeext here because the later test for it is not # safe: cross compilers may not add the suffix if given an `-o' # argument, so we may need to know it at that point already. # Even if this section looks crufty: it has the advantage of # actually working. break;; * ) break;; esac done test "$ac_cv_exeext" = no && ac_cv_exeext= else ac_file='' fi if test -z "$ac_file"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error 77 "C compiler cannot create executables See \`config.log' for more details" "$LINENO" 5; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5 $as_echo_n "checking for C compiler default output file name... " >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5 $as_echo "$ac_file" >&6; } ac_exeext=$ac_cv_exeext rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out ac_clean_files=$ac_clean_files_save { $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5 $as_echo_n "checking for suffix of executables... " >&6; } if { { ac_try="$ac_link" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then : # If both `conftest.exe' and `conftest' are `present' (well, observable) # catch `conftest.exe'. For instance with Cygwin, `ls conftest' will # work properly (i.e., refer to `conftest.exe'), while it won't with # `rm'. for ac_file in conftest.exe conftest conftest.*; do test -f "$ac_file" || continue case $ac_file in *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` break;; * ) break;; esac done else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of executables: cannot compile and link See \`config.log' for more details" "$LINENO" 5; } fi rm -f conftest conftest$ac_cv_exeext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 $as_echo "$ac_cv_exeext" >&6; } rm -f conftest.$ac_ext EXEEXT=$ac_cv_exeext ac_exeext=$EXEEXT cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { FILE *f = fopen ("conftest.out", "w"); return ferror (f) || fclose (f) != 0; ; return 0; } _ACEOF ac_clean_files="$ac_clean_files conftest.out" # Check that the compiler produces executables we can run. If not, either # the compiler is broken, or we cross compile. { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5 $as_echo_n "checking whether we are cross compiling... " >&6; } if test "$cross_compiling" != yes; then { { ac_try="$ac_link" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } if { ac_try='./conftest$ac_cv_exeext' { { case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_try") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; }; then cross_compiling=no else if test "$cross_compiling" = maybe; then cross_compiling=yes else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "cannot run C compiled programs. If you meant to cross compile, use \`--host'. See \`config.log' for more details" "$LINENO" 5; } fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 $as_echo "$cross_compiling" >&6; } rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out ac_clean_files=$ac_clean_files_save { $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 $as_echo_n "checking for suffix of object files... " >&6; } if ${ac_cv_objext+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF rm -f conftest.o conftest.obj if { { ac_try="$ac_compile" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_compile") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then : for ac_file in conftest.o conftest.obj conftest.*; do test -f "$ac_file" || continue; case $ac_file in *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;; *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'` break;; esac done else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of object files: cannot compile See \`config.log' for more details" "$LINENO" 5; } fi rm -f conftest.$ac_cv_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 $as_echo "$ac_cv_objext" >&6; } OBJEXT=$ac_cv_objext ac_objext=$OBJEXT { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5 $as_echo_n "checking whether we are using the GNU C compiler... " >&6; } if ${ac_cv_c_compiler_gnu+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { #ifndef __GNUC__ choke me #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_compiler_gnu=yes else ac_compiler_gnu=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_cv_c_compiler_gnu=$ac_compiler_gnu fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 $as_echo "$ac_cv_c_compiler_gnu" >&6; } if test $ac_compiler_gnu = yes; then GCC=yes else GCC= fi ac_test_CFLAGS=${CFLAGS+set} ac_save_CFLAGS=$CFLAGS { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 $as_echo_n "checking whether $CC accepts -g... " >&6; } if ${ac_cv_prog_cc_g+:} false; then : $as_echo_n "(cached) " >&6 else ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes ac_cv_prog_cc_g=no CFLAGS="-g" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_prog_cc_g=yes else CFLAGS="" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : else ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="-g" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_prog_cc_g=yes fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 $as_echo "$ac_cv_prog_cc_g" >&6; } if test "$ac_test_CFLAGS" = set; then CFLAGS=$ac_save_CFLAGS elif test $ac_cv_prog_cc_g = yes; then if test "$GCC" = yes; then CFLAGS="-g -O2" else CFLAGS="-g" fi else if test "$GCC" = yes; then CFLAGS="-O2" else CFLAGS= fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5 $as_echo_n "checking for $CC option to accept ISO C89... " >&6; } if ${ac_cv_prog_cc_c89+:} false; then : $as_echo_n "(cached) " >&6 else ac_cv_prog_cc_c89=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include struct stat; /* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ struct buf { int x; }; FILE * (*rcsopen) (struct buf *, struct stat *, int); static char *e (p, i) char **p; int i; { return p[i]; } static char *f (char * (*g) (char **, int), char **p, ...) { char *s; va_list v; va_start (v,p); s = g (p, va_arg (v,int)); va_end (v); return s; } /* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has function prototypes and stuff, but not '\xHH' hex character constants. These don't provoke an error unfortunately, instead are silently treated as 'x'. The following induces an error, until -std is added to get proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an array size at least. It's necessary to write '\x00'==0 to get something that's true only with -std. */ int osf4_cc_array ['\x00' == 0 ? 1 : -1]; /* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters inside strings and character constants. */ #define FOO(x) 'x' int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1]; int test (int i, double x); struct s1 {int (*f) (int a);}; struct s2 {int (*f) (double a);}; int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); int argc; char **argv; int main () { return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; ; return 0; } _ACEOF for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \ -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" do CC="$ac_save_CC $ac_arg" if ac_fn_c_try_compile "$LINENO"; then : ac_cv_prog_cc_c89=$ac_arg fi rm -f core conftest.err conftest.$ac_objext test "x$ac_cv_prog_cc_c89" != "xno" && break done rm -f conftest.$ac_ext CC=$ac_save_CC fi # AC_CACHE_VAL case "x$ac_cv_prog_cc_c89" in x) { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 $as_echo "none needed" >&6; } ;; xno) { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 $as_echo "unsupported" >&6; } ;; *) CC="$CC $ac_cv_prog_cc_c89" { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 $as_echo "$ac_cv_prog_cc_c89" >&6; } ;; esac if test "x$ac_cv_prog_cc_c89" != xno; then : fi ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC understands -c and -o together" >&5 $as_echo_n "checking whether $CC understands -c and -o together... " >&6; } if ${am_cv_prog_cc_c_o+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF # Make sure it works both with $CC and with simple cc. # Following AC_PROG_CC_C_O, we do the test twice because some # compilers refuse to overwrite an existing .o file with -o, # though they will create one. am_cv_prog_cc_c_o=yes for am_i in 1 2; do if { echo "$as_me:$LINENO: $CC -c conftest.$ac_ext -o conftest2.$ac_objext" >&5 ($CC -c conftest.$ac_ext -o conftest2.$ac_objext) >&5 2>&5 ac_status=$? echo "$as_me:$LINENO: \$? = $ac_status" >&5 (exit $ac_status); } \ && test -f conftest2.$ac_objext; then : OK else am_cv_prog_cc_c_o=no break fi done rm -f core conftest* unset am_i fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_prog_cc_c_o" >&5 $as_echo "$am_cv_prog_cc_c_o" >&6; } if test "$am_cv_prog_cc_c_o" != yes; then # Losing compiler, so override with the script. # FIXME: It is wrong to rewrite CC. # But if we don't then we get into trouble of one sort or another. # A longer-term fix would be to have automake use am__CC in this case, # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" CC="$am_aux_dir/compile $CC" fi ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu DEPDIR="${am__leading_dot}deps" ac_config_commands="$ac_config_commands depfiles" am_make=${MAKE-make} cat > confinc << 'END' am__doit: @echo this is the am__doit target .PHONY: am__doit END # If we don't find an include directive, just comment out the code. { $as_echo "$as_me:${as_lineno-$LINENO}: checking for style of include used by $am_make" >&5 $as_echo_n "checking for style of include used by $am_make... " >&6; } am__include="#" am__quote= _am_result=none # First try GNU make style include. echo "include confinc" > confmf # Ignore all kinds of additional output from 'make'. case `$am_make -s -f confmf 2> /dev/null` in #( *the\ am__doit\ target*) am__include=include am__quote= _am_result=GNU ;; esac # Now try BSD make style include. if test "$am__include" = "#"; then echo '.include "confinc"' > confmf case `$am_make -s -f confmf 2> /dev/null` in #( *the\ am__doit\ target*) am__include=.include am__quote="\"" _am_result=BSD ;; esac fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $_am_result" >&5 $as_echo "$_am_result" >&6; } rm -f confinc confmf # Check whether --enable-dependency-tracking was given. if test "${enable_dependency_tracking+set}" = set; then : enableval=$enable_dependency_tracking; fi if test "x$enable_dependency_tracking" != xno; then am_depcomp="$ac_aux_dir/depcomp" AMDEPBACKSLASH='\' am__nodep='_no' fi if test "x$enable_dependency_tracking" != xno; then AMDEP_TRUE= AMDEP_FALSE='#' else AMDEP_TRUE='#' AMDEP_FALSE= fi depcc="$CC" am_compiler_list= { $as_echo "$as_me:${as_lineno-$LINENO}: checking dependency style of $depcc" >&5 $as_echo_n "checking dependency style of $depcc... " >&6; } if ${am_cv_CC_dependencies_compiler_type+:} false; then : $as_echo_n "(cached) " >&6 else if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then # We make a subdir and do the tests there. Otherwise we can end up # making bogus files that we don't know about and never remove. For # instance it was reported that on HP-UX the gcc test will end up # making a dummy file named 'D' -- because '-MD' means "put the output # in D". rm -rf conftest.dir mkdir conftest.dir # Copy depcomp to subdir because otherwise we won't find it if we're # using a relative directory. cp "$am_depcomp" conftest.dir cd conftest.dir # We will build objects and dependencies in a subdirectory because # it helps to detect inapplicable dependency modes. For instance # both Tru64's cc and ICC support -MD to output dependencies as a # side effect of compilation, but ICC will put the dependencies in # the current directory while Tru64 will put them in the object # directory. mkdir sub am_cv_CC_dependencies_compiler_type=none if test "$am_compiler_list" = ""; then am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` fi am__universal=false case " $depcc " in #( *\ -arch\ *\ -arch\ *) am__universal=true ;; esac for depmode in $am_compiler_list; do # Setup a source with many dependencies, because some compilers # like to wrap large dependency lists on column 80 (with \), and # we should not choose a depcomp mode which is confused by this. # # We need to recreate these files for each test, as the compiler may # overwrite some of them when testing with obscure command lines. # This happens at least with the AIX C compiler. : > sub/conftest.c for i in 1 2 3 4 5 6; do echo '#include "conftst'$i'.h"' >> sub/conftest.c # Using ": > sub/conftst$i.h" creates only sub/conftst1.h with # Solaris 10 /bin/sh. echo '/* dummy */' > sub/conftst$i.h done echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf # We check with '-c' and '-o' for the sake of the "dashmstdout" # mode. It turns out that the SunPro C++ compiler does not properly # handle '-M -o', and we need to detect this. Also, some Intel # versions had trouble with output in subdirs. am__obj=sub/conftest.${OBJEXT-o} am__minus_obj="-o $am__obj" case $depmode in gcc) # This depmode causes a compiler race in universal mode. test "$am__universal" = false || continue ;; nosideeffect) # After this tag, mechanisms are not by side-effect, so they'll # only be used when explicitly requested. if test "x$enable_dependency_tracking" = xyes; then continue else break fi ;; msvc7 | msvc7msys | msvisualcpp | msvcmsys) # This compiler won't grok '-c -o', but also, the minuso test has # not run yet. These depmodes are late enough in the game, and # so weak that their functioning should not be impacted. am__obj=conftest.${OBJEXT-o} am__minus_obj= ;; none) break ;; esac if depmode=$depmode \ source=sub/conftest.c object=$am__obj \ depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ >/dev/null 2>conftest.err && grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && grep $am__obj sub/conftest.Po > /dev/null 2>&1 && ${MAKE-make} -s -f confmf > /dev/null 2>&1; then # icc doesn't choke on unknown options, it will just issue warnings # or remarks (even with -Werror). So we grep stderr for any message # that says an option was ignored or not supported. # When given -MP, icc 7.0 and 7.1 complain thusly: # icc: Command line warning: ignoring option '-M'; no argument required # The diagnosis changed in icc 8.0: # icc: Command line remark: option '-MP' not supported if (grep 'ignoring option' conftest.err || grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else am_cv_CC_dependencies_compiler_type=$depmode break fi fi done cd .. rm -rf conftest.dir else am_cv_CC_dependencies_compiler_type=none fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_CC_dependencies_compiler_type" >&5 $as_echo "$am_cv_CC_dependencies_compiler_type" >&6; } CCDEPMODE=depmode=$am_cv_CC_dependencies_compiler_type if test "x$enable_dependency_tracking" != xno \ && test "$am_cv_CC_dependencies_compiler_type" = gcc3; then am__fastdepCC_TRUE= am__fastdepCC_FALSE='#' else am__fastdepCC_TRUE='#' am__fastdepCC_FALSE= fi ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu { $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 $as_echo_n "checking how to run the C preprocessor... " >&6; } # On Suns, sometimes $CPP names a directory. if test -n "$CPP" && test -d "$CPP"; then CPP= fi if test -z "$CPP"; then if ${ac_cv_prog_CPP+:} false; then : $as_echo_n "(cached) " >&6 else # Double quotes because CPP needs to be expanded for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp" do ac_preproc_ok=false for ac_c_preproc_warn_flag in '' yes do # Use a header file that comes with gcc, so configuring glibc # with a fresh cross-compiler works. # Prefer to if __STDC__ is defined, since # exists even on freestanding compilers. # On the NeXT, cc -E runs the code through the compiler's parser, # not just through cpp. "Syntax error" is here to catch this case. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef __STDC__ # include #else # include #endif Syntax error _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : else # Broken: fails on valid input. continue fi rm -f conftest.err conftest.i conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : # Broken: success on invalid input. continue else # Passes both tests. ac_preproc_ok=: break fi rm -f conftest.err conftest.i conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok; then : break fi done ac_cv_prog_CPP=$CPP fi CPP=$ac_cv_prog_CPP else ac_cv_prog_CPP=$CPP fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CPP" >&5 $as_echo "$CPP" >&6; } ac_preproc_ok=false for ac_c_preproc_warn_flag in '' yes do # Use a header file that comes with gcc, so configuring glibc # with a fresh cross-compiler works. # Prefer to if __STDC__ is defined, since # exists even on freestanding compilers. # On the NeXT, cc -E runs the code through the compiler's parser, # not just through cpp. "Syntax error" is here to catch this case. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef __STDC__ # include #else # include #endif Syntax error _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : else # Broken: fails on valid input. continue fi rm -f conftest.err conftest.i conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : # Broken: success on invalid input. continue else # Passes both tests. ac_preproc_ok=: break fi rm -f conftest.err conftest.i conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok; then : else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "C preprocessor \"$CPP\" fails sanity check See \`config.log' for more details" "$LINENO" 5; } fi ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu { $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 $as_echo_n "checking for grep that handles long lines and -e... " >&6; } if ${ac_cv_path_GREP+:} false; then : $as_echo_n "(cached) " >&6 else if test -z "$GREP"; then ac_path_GREP_found=false # Loop through the user's path and test for each of PROGNAME-LIST as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_prog in grep ggrep; do for ac_exec_ext in '' $ac_executable_extensions; do ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext" as_fn_executable_p "$ac_path_GREP" || continue # Check for GNU ac_path_GREP and select it if it is found. # Check for GNU $ac_path_GREP case `"$ac_path_GREP" --version 2>&1` in *GNU*) ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; *) ac_count=0 $as_echo_n 0123456789 >"conftest.in" while : do cat "conftest.in" "conftest.in" >"conftest.tmp" mv "conftest.tmp" "conftest.in" cp "conftest.in" "conftest.nl" $as_echo 'GREP' >> "conftest.nl" "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break as_fn_arith $ac_count + 1 && ac_count=$as_val if test $ac_count -gt ${ac_path_GREP_max-0}; then # Best one so far, save it but keep looking for a better one ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_max=$ac_count fi # 10*(2^10) chars as input seems more than enough test $ac_count -gt 10 && break done rm -f conftest.in conftest.tmp conftest.nl conftest.out;; esac $ac_path_GREP_found && break 3 done done done IFS=$as_save_IFS if test -z "$ac_cv_path_GREP"; then as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_GREP=$GREP fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5 $as_echo "$ac_cv_path_GREP" >&6; } GREP="$ac_cv_path_GREP" { $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 $as_echo_n "checking for egrep... " >&6; } if ${ac_cv_path_EGREP+:} false; then : $as_echo_n "(cached) " >&6 else if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 then ac_cv_path_EGREP="$GREP -E" else if test -z "$EGREP"; then ac_path_EGREP_found=false # Loop through the user's path and test for each of PROGNAME-LIST as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_prog in egrep; do for ac_exec_ext in '' $ac_executable_extensions; do ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext" as_fn_executable_p "$ac_path_EGREP" || continue # Check for GNU ac_path_EGREP and select it if it is found. # Check for GNU $ac_path_EGREP case `"$ac_path_EGREP" --version 2>&1` in *GNU*) ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; *) ac_count=0 $as_echo_n 0123456789 >"conftest.in" while : do cat "conftest.in" "conftest.in" >"conftest.tmp" mv "conftest.tmp" "conftest.in" cp "conftest.in" "conftest.nl" $as_echo 'EGREP' >> "conftest.nl" "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break as_fn_arith $ac_count + 1 && ac_count=$as_val if test $ac_count -gt ${ac_path_EGREP_max-0}; then # Best one so far, save it but keep looking for a better one ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_max=$ac_count fi # 10*(2^10) chars as input seems more than enough test $ac_count -gt 10 && break done rm -f conftest.in conftest.tmp conftest.nl conftest.out;; esac $ac_path_EGREP_found && break 3 done done done IFS=$as_save_IFS if test -z "$ac_cv_path_EGREP"; then as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_EGREP=$EGREP fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 $as_echo "$ac_cv_path_EGREP" >&6; } EGREP="$ac_cv_path_EGREP" ICC="no" { $as_echo "$as_me:${as_lineno-$LINENO}: checking for icc in use" >&5 $as_echo_n "checking for icc in use... " >&6; } if test "$GCC" = "yes"; then cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ __INTEL_COMPILER _ACEOF if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | $EGREP "^__INTEL_COMPILER" >/dev/null 2>&1; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } ICC="yes" fi rm -f conftest* else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for clang in use" >&5 $as_echo_n "checking for clang in use... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { #ifndef __clang__ not clang #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : CLANG=yes else CLANG=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CLANG" >&5 $as_echo "$CLANG" >&6; } if test "$CLANG" = "yes"; then : fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 $as_echo_n "checking for ANSI C header files... " >&6; } if ${ac_cv_header_stdc+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #include #include int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_header_stdc=yes else ac_cv_header_stdc=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext if test $ac_cv_header_stdc = yes; then # SunOS 4.x string.h does not declare mem*, contrary to ANSI. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include _ACEOF if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | $EGREP "memchr" >/dev/null 2>&1; then : else ac_cv_header_stdc=no fi rm -f conftest* fi if test $ac_cv_header_stdc = yes; then # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include _ACEOF if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | $EGREP "free" >/dev/null 2>&1; then : else ac_cv_header_stdc=no fi rm -f conftest* fi if test $ac_cv_header_stdc = yes; then # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. if test "$cross_compiling" = yes; then : : else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #if ((' ' & 0x0FF) == 0x020) # define ISLOWER(c) ('a' <= (c) && (c) <= 'z') # define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) #else # define ISLOWER(c) \ (('a' <= (c) && (c) <= 'i') \ || ('j' <= (c) && (c) <= 'r') \ || ('s' <= (c) && (c) <= 'z')) # define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) #endif #define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) int main () { int i; for (i = 0; i < 256; i++) if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) return 2; return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : else ac_cv_header_stdc=no fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5 $as_echo "$ac_cv_header_stdc" >&6; } if test $ac_cv_header_stdc = yes; then $as_echo "#define STDC_HEADERS 1" >>confdefs.h fi # On IRIX 5.3, sys/types and inttypes.h are conflicting. for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \ inttypes.h stdint.h unistd.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default " if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 _ACEOF fi done ac_fn_c_check_decl "$LINENO" "__SUNPRO_C" "ac_cv_have_decl___SUNPRO_C" "$ac_includes_default" if test "x$ac_cv_have_decl___SUNPRO_C" = xyes; then : SUNCC="yes" else SUNCC="no" fi if test "x$SUNCC" = "xyes"; then : CFLAGS="-mt $CFLAGS" fi if test "$ICC" = "yes" -o "$GCC" = "yes"; then : if test "$CLANG" = "no"; then : CFLAGS="$CFLAGS -pthread" fi fi if test "$ICC" = "no"; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C99" >&5 $as_echo_n "checking for $CC option to accept ISO C99... " >&6; } if ${ac_cv_prog_cc_c99+:} false; then : $as_echo_n "(cached) " >&6 else ac_cv_prog_cc_c99=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #include #include #include // Check varargs macros. These examples are taken from C99 6.10.3.5. #define debug(...) fprintf (stderr, __VA_ARGS__) #define showlist(...) puts (#__VA_ARGS__) #define report(test,...) ((test) ? puts (#test) : printf (__VA_ARGS__)) static void test_varargs_macros (void) { int x = 1234; int y = 5678; debug ("Flag"); debug ("X = %d\n", x); showlist (The first, second, and third items.); report (x>y, "x is %d but y is %d", x, y); } // Check long long types. #define BIG64 18446744073709551615ull #define BIG32 4294967295ul #define BIG_OK (BIG64 / BIG32 == 4294967297ull && BIG64 % BIG32 == 0) #if !BIG_OK your preprocessor is broken; #endif #if BIG_OK #else your preprocessor is broken; #endif static long long int bignum = -9223372036854775807LL; static unsigned long long int ubignum = BIG64; struct incomplete_array { int datasize; double data[]; }; struct named_init { int number; const wchar_t *name; double average; }; typedef const char *ccp; static inline int test_restrict (ccp restrict text) { // See if C++-style comments work. // Iterate through items via the restricted pointer. // Also check for declarations in for loops. for (unsigned int i = 0; *(text+i) != '\0'; ++i) continue; return 0; } // Check varargs and va_copy. static void test_varargs (const char *format, ...) { va_list args; va_start (args, format); va_list args_copy; va_copy (args_copy, args); const char *str; int number; float fnumber; while (*format) { switch (*format++) { case 's': // string str = va_arg (args_copy, const char *); break; case 'd': // int number = va_arg (args_copy, int); break; case 'f': // float fnumber = va_arg (args_copy, double); break; default: break; } } va_end (args_copy); va_end (args); } int main () { // Check bool. _Bool success = false; // Check restrict. if (test_restrict ("String literal") == 0) success = true; char *restrict newvar = "Another string"; // Check varargs. test_varargs ("s, d' f .", "string", 65, 34.234); test_varargs_macros (); // Check flexible array members. struct incomplete_array *ia = malloc (sizeof (struct incomplete_array) + (sizeof (double) * 10)); ia->datasize = 10; for (int i = 0; i < ia->datasize; ++i) ia->data[i] = i * 1.234; // Check named initializers. struct named_init ni = { .number = 34, .name = L"Test wide string", .average = 543.34343, }; ni.number = 58; int dynamic_array[ni.number]; dynamic_array[ni.number - 1] = 543; // work around unused variable warnings return (!success || bignum == 0LL || ubignum == 0uLL || newvar[0] == 'x' || dynamic_array[ni.number - 1] != 543); ; return 0; } _ACEOF for ac_arg in '' -std=gnu99 -c99 -qlanglvl=extc99 do CC="$ac_save_CC $ac_arg" if ac_fn_c_try_compile "$LINENO"; then : ac_cv_prog_cc_c99=$ac_arg fi rm -f core conftest.err conftest.$ac_objext test "x$ac_cv_prog_cc_c99" != "xno" && break done rm -f conftest.$ac_ext CC=$ac_save_CC fi # AC_CACHE_VAL case "x$ac_cv_prog_cc_c99" in x) { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 $as_echo "none needed" >&6; } ;; xno) { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 $as_echo "unsupported" >&6; } ;; *) CC="$CC $ac_cv_prog_cc_c99" { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5 $as_echo "$ac_cv_prog_cc_c99" >&6; } ;; esac if test "x$ac_cv_prog_cc_c99" != xno; then : fi fi # Check whether --enable-arm_crc32 was given. if test "${enable_arm_crc32+set}" = set; then : enableval=$enable_arm_crc32; fi # Check whether --enable-extstore was given. if test "${enable_extstore+set}" = set; then : enableval=$enable_extstore; fi # Check whether --enable-seccomp was given. if test "${enable_seccomp+set}" = set; then : enableval=$enable_seccomp; fi # Check whether --enable-sasl was given. if test "${enable_sasl+set}" = set; then : enableval=$enable_sasl; fi # Check whether --enable-sasl_pwdb was given. if test "${enable_sasl_pwdb+set}" = set; then : enableval=$enable_sasl_pwdb; fi if test "x$enable_sasl_pwdb" = "xyes"; then : enable_sasl=yes fi # Check whether --enable-tls was given. if test "${enable_tls+set}" = set; then : enableval=$enable_tls; fi # Check whether --enable-asan was given. if test "${enable_asan+set}" = set; then : enableval=$enable_asan; fi for ac_header in sasl/sasl.h do : ac_fn_c_check_header_mongrel "$LINENO" "sasl/sasl.h" "ac_cv_header_sasl_sasl_h" "$ac_includes_default" if test "x$ac_cv_header_sasl_sasl_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_SASL_SASL_H 1 _ACEOF fi done if test "x$enable_sasl" = "xyes"; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SASL_CB_GETCONF" >&5 $as_echo_n "checking for SASL_CB_GETCONF... " >&6; } if ${ac_cv_c_sasl_cb_getconf+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { unsigned long val = SASL_CB_GETCONF; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_c_sasl_cb_getconf=yes else ac_cv_c_sasl_cb_getconf=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_sasl_cb_getconf" >&5 $as_echo "$ac_cv_c_sasl_cb_getconf" >&6; } if test "$ac_cv_c_sasl_cb_getconf" = "yes"; then : $as_echo "#define HAVE_SASL_CB_GETCONF 1" >>confdefs.h fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SASL_CB_GETCONFPATH" >&5 $as_echo_n "checking for SASL_CB_GETCONFPATH... " >&6; } if ${ac_cv_c_sasl_cb_getconfpath+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { unsigned long val = SASL_CB_GETCONFPATH; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_c_sasl_cb_getconfpath=yes else ac_cv_c_sasl_cb_getconfpath=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_sasl_cb_getconfpath" >&5 $as_echo "$ac_cv_c_sasl_cb_getconfpath" >&6; } if test "$ac_cv_c_sasl_cb_getconfpath" = "yes"; then : $as_echo "#define HAVE_SASL_CB_GETCONFPATH 1" >>confdefs.h fi $as_echo "#define ENABLE_SASL 1" >>confdefs.h { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing sasl_server_init" >&5 $as_echo_n "checking for library containing sasl_server_init... " >&6; } if ${ac_cv_search_sasl_server_init+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char sasl_server_init (); int main () { return sasl_server_init (); ; return 0; } _ACEOF for ac_lib in '' sasl2 sasl; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_sasl_server_init=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_sasl_server_init+:} false; then : break fi done if ${ac_cv_search_sasl_server_init+:} false; then : else ac_cv_search_sasl_server_init=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_sasl_server_init" >&5 $as_echo "$ac_cv_search_sasl_server_init" >&6; } ac_res=$ac_cv_search_sasl_server_init if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" else as_fn_error $? "Failed to locate the library containing sasl_server_init" "$LINENO" 5 fi if test "x$enable_sasl_pwdb" = "xyes"; then : $as_echo "#define ENABLE_SASL_PWDB 1" >>confdefs.h fi fi # Check whether --enable-dtrace was given. if test "${enable_dtrace+set}" = set; then : enableval=$enable_dtrace; fi if test "x$enable_dtrace" = "xyes"; then # Extract the first word of "dtrace", so it can be a program name with args. set dummy dtrace; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_DTRACE+:} false; then : $as_echo_n "(cached) " >&6 else case $DTRACE in [\\/]* | ?:[\\/]*) ac_cv_path_DTRACE="$DTRACE" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR as_dummy="/usr/sbin:$PATH" for as_dir in $as_dummy do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_DTRACE="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS test -z "$ac_cv_path_DTRACE" && ac_cv_path_DTRACE=""no"" ;; esac fi DTRACE=$ac_cv_path_DTRACE if test -n "$DTRACE"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DTRACE" >&5 $as_echo "$DTRACE" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi if test "x$DTRACE" != "xno"; then $as_echo "#define ENABLE_DTRACE 1" >>confdefs.h build_dtrace=yes $DTRACE -h -o conftest.h -s memcached_dtrace.d 2>/dev/zero if test $? -eq 0 then dtrace_instrument_obj=yes rm conftest.h fi if test "`which tr`" = "/usr/ucb/tr"; then as_fn_error $? "Please remove /usr/ucb from your path. See man standards for more info" "$LINENO" 5 fi else as_fn_error $? "Need dtrace binary and OS support." "$LINENO" 5 fi fi if test "x$enable_extstore" = "xyes"; then $as_echo "#define EXTSTORE 1" >>confdefs.h fi if test "x$enable_tls" = "xyes"; then $as_echo "#define TLS 1" >>confdefs.h fi if test "x$enable_arm_crc32" = "xyes"; then $as_echo "#define ARM_CRC32 1" >>confdefs.h fi if test "x$enable_asan" = "xyes"; then $as_echo "#define ASAN 1" >>confdefs.h fi if test "$build_dtrace" = "yes"; then BUILD_DTRACE_TRUE= BUILD_DTRACE_FALSE='#' else BUILD_DTRACE_TRUE='#' BUILD_DTRACE_FALSE= fi if test "$dtrace_instrument_obj" = "yes"; then DTRACE_INSTRUMENT_OBJ_TRUE= DTRACE_INSTRUMENT_OBJ_FALSE='#' else DTRACE_INSTRUMENT_OBJ_TRUE='#' DTRACE_INSTRUMENT_OBJ_FALSE= fi if test "$enable_sasl" = "yes"; then ENABLE_SASL_TRUE= ENABLE_SASL_FALSE='#' else ENABLE_SASL_TRUE='#' ENABLE_SASL_FALSE= fi if test "$enable_extstore" = "yes"; then ENABLE_EXTSTORE_TRUE= ENABLE_EXTSTORE_FALSE='#' else ENABLE_EXTSTORE_TRUE='#' ENABLE_EXTSTORE_FALSE= fi if test "$enable_arm_crc32" = "yes"; then ENABLE_ARM_CRC32_TRUE= ENABLE_ARM_CRC32_FALSE='#' else ENABLE_ARM_CRC32_TRUE='#' ENABLE_ARM_CRC32_FALSE= fi if test "$enable_tls" = "yes"; then ENABLE_TLS_TRUE= ENABLE_TLS_FALSE='#' else ENABLE_TLS_TRUE='#' ENABLE_TLS_FALSE= fi if test "$enable_asan" = "yes"; then ENABLE_ASAN_TRUE= ENABLE_ASAN_FALSE='#' else ENABLE_ASAN_TRUE='#' ENABLE_ASAN_FALSE= fi # Check whether --enable-coverage was given. if test "${enable_coverage+set}" = set; then : enableval=$enable_coverage; fi if test "x$enable_coverage" != "xno"; then if test "$GCC" = "yes" -a "$ICC" != "yes" -a "$CLANG" != "yes" then CFLAGS="$CFLAGS -pthread" # Extract the first word of "gcov", so it can be a program name with args. set dummy gcov; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_PROFILER+:} false; then : $as_echo_n "(cached) " >&6 else case $PROFILER in [\\/]* | ?:[\\/]*) ac_cv_path_PROFILER="$PROFILER" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_PROFILER="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS test -z "$ac_cv_path_PROFILER" && ac_cv_path_PROFILER=""no"" ;; esac fi PROFILER=$ac_cv_path_PROFILER if test -n "$PROFILER"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PROFILER" >&5 $as_echo "$PROFILER" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi if test "x$PROFILER" != "xno"; then # Issue 97: The existence of gcov doesn't mean we have -lgcov { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lgcov" >&5 $as_echo_n "checking for main in -lgcov... " >&6; } if ${ac_cv_lib_gcov_main+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS LIBS="-lgcov $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { return main (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : ac_cv_lib_gcov_main=yes else ac_cv_lib_gcov_main=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gcov_main" >&5 $as_echo "$ac_cv_lib_gcov_main" >&6; } if test "x$ac_cv_lib_gcov_main" = xyes; then : PROFILER_FLAGS="-fprofile-arcs -ftest-coverage" PROFILER_LDFLAGS="-lgcov" else PROFILER_FLAGS= PROFILER_LDFLAGS= fi fi elif test "$SUNCC" = "yes" then # Extract the first word of "tcov", so it can be a program name with args. set dummy tcov; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_PROFILER+:} false; then : $as_echo_n "(cached) " >&6 else case $PROFILER in [\\/]* | ?:[\\/]*) ac_cv_path_PROFILER="$PROFILER" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_PROFILER="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS test -z "$ac_cv_path_PROFILER" && ac_cv_path_PROFILER=""no"" ;; esac fi PROFILER=$ac_cv_path_PROFILER if test -n "$PROFILER"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PROFILER" >&5 $as_echo "$PROFILER" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi if test "x$PROFILER" != "xno"; then PROFILER_FLAGS=-xprofile=tcov fi elif test "x$CLANG" != "xno" then # Extract the first word of "gcov", so it can be a program name with args. set dummy gcov; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_PROFILER+:} false; then : $as_echo_n "(cached) " >&6 else case $PROFILER in [\\/]* | ?:[\\/]*) ac_cv_path_PROFILER="$PROFILER" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_PROFILER="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS test -z "$ac_cv_path_PROFILER" && ac_cv_path_PROFILER=""no"" ;; esac fi PROFILER=$ac_cv_path_PROFILER if test -n "$PROFILER"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PROFILER" >&5 $as_echo "$PROFILER" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi if test "x$PROFILER" != "xno" then PROFILER_FLAGS="-fprofile-arcs -ftest-coverage" PROFILER_LDFLAGS= fi fi fi # Check whether --enable-64bit was given. if test "${enable_64bit+set}" = set; then : enableval=$enable_64bit; fi if test "x$enable_64bit" = "xyes" then org_cflags=$CFLAGS CFLAGS=-m64 if test "$cross_compiling" = yes; then : { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Assuming no extra CFLAGS are required for cross-compiling 64bit version." >&5 $as_echo "$as_me: WARNING: Assuming no extra CFLAGS are required for cross-compiling 64bit version." >&2;} else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { return sizeof(void*) == 8 ? 0 : 1; ; return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : CFLAGS="-m64 $org_cflags" else as_fn_error $? "Don't know how to build a 64-bit object." "$LINENO" 5 fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi fi # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of void *" >&5 $as_echo_n "checking size of void *... " >&6; } if ${ac_cv_sizeof_void_p+:} false; then : $as_echo_n "(cached) " >&6 else if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (void *))" "ac_cv_sizeof_void_p" "$ac_includes_default"; then : else if test "$ac_cv_type_void_p" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (void *) See \`config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_void_p=0 fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_void_p" >&5 $as_echo "$ac_cv_sizeof_void_p" >&6; } cat >>confdefs.h <<_ACEOF #define SIZEOF_VOID_P $ac_cv_sizeof_void_p _ACEOF # Issue 213: Search for clock_gettime to help people linking # with a static version of libevent { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing clock_gettime" >&5 $as_echo_n "checking for library containing clock_gettime... " >&6; } if ${ac_cv_search_clock_gettime+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char clock_gettime (); int main () { return clock_gettime (); ; return 0; } _ACEOF for ac_lib in '' rt; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_clock_gettime=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_clock_gettime+:} false; then : break fi done if ${ac_cv_search_clock_gettime+:} false; then : else ac_cv_search_clock_gettime=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_clock_gettime" >&5 $as_echo "$ac_cv_search_clock_gettime" >&6; } ac_res=$ac_cv_search_clock_gettime if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi # Issue 214: Search for the network libraries _before_ searching # for libevent (to help people linking with static libevent) { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing socket" >&5 $as_echo_n "checking for library containing socket... " >&6; } if ${ac_cv_search_socket+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char socket (); int main () { return socket (); ; return 0; } _ACEOF for ac_lib in '' socket; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_socket=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_socket+:} false; then : break fi done if ${ac_cv_search_socket+:} false; then : else ac_cv_search_socket=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_socket" >&5 $as_echo "$ac_cv_search_socket" >&6; } ac_res=$ac_cv_search_socket if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing gethostbyname" >&5 $as_echo_n "checking for library containing gethostbyname... " >&6; } if ${ac_cv_search_gethostbyname+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char gethostbyname (); int main () { return gethostbyname (); ; return 0; } _ACEOF for ac_lib in '' nsl; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_gethostbyname=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_gethostbyname+:} false; then : break fi done if ${ac_cv_search_gethostbyname+:} false; then : else ac_cv_search_gethostbyname=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_gethostbyname" >&5 $as_echo "$ac_cv_search_gethostbyname" >&6; } ac_res=$ac_cv_search_gethostbyname if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi trylibeventdir="" # Check whether --with-libevent was given. if test "${with_libevent+set}" = set; then : withval=$with_libevent; if test "x$withval" != "xno" ; then trylibeventdir=$withval fi fi LIBEVENT_URL=https://www.monkey.org/~provos/libevent/ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for libevent directory" >&5 $as_echo_n "checking for libevent directory... " >&6; } if ${ac_cv_libevent_dir+:} false; then : $as_echo_n "(cached) " >&6 else saved_LIBS="$LIBS" saved_LDFLAGS="$LDFLAGS" saved_CPPFLAGS="$CPPFLAGS" le_found=no for ledir in $trylibeventdir "" $prefix /usr/local ; do LDFLAGS="$saved_LDFLAGS" LIBS="-levent $saved_LIBS" # Skip the directory if it isn't there. if test ! -z "$ledir" -a ! -d "$ledir" ; then continue; fi if test ! -z "$ledir" ; then if test -d "$ledir/lib" ; then LDFLAGS="-L$ledir/lib $LDFLAGS" else LDFLAGS="-L$ledir $LDFLAGS" fi if test -d "$ledir/include" ; then CPPFLAGS="-I$ledir/include $CPPFLAGS" else CPPFLAGS="-I$ledir $CPPFLAGS" fi fi # Can I compile and link it? cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #include int main () { event_init(); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : libevent_linked=yes else libevent_linked=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext if test $libevent_linked = yes; then if test ! -z "$ledir" ; then ac_cv_libevent_dir=$ledir _myos=`echo $target_os | cut -f 1 -d .` if test "$SUNCC" = "yes" -o "x$_myos" = "xsolaris2"; then : saved_LDFLAGS="$saved_LDFLAGS -Wl,-R$ledir/lib" else if test "$GCC" = "yes"; then : saved_LDFLAGS="$saved_LDFLAGS -Wl,-rpath,$ledir/lib" fi fi else ac_cv_libevent_dir="(system)" fi le_found=yes break fi done LIBS="$saved_LIBS" LDFLAGS="$saved_LDFLAGS" CPPFLAGS="$saved_CPPFLAGS" if test $le_found = no ; then as_fn_error $? "libevent is required. You can get it from $LIBEVENT_URL If it's already installed, specify its path using --with-libevent=/dir/ " "$LINENO" 5 fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libevent_dir" >&5 $as_echo "$ac_cv_libevent_dir" >&6; } LIBS="-levent $LIBS" if test $ac_cv_libevent_dir != "(system)"; then if test -d "$ac_cv_libevent_dir/lib" ; then LDFLAGS="-L$ac_cv_libevent_dir/lib $LDFLAGS" le_libdir="$ac_cv_libevent_dir/lib" else LDFLAGS="-L$ac_cv_libevent_dir $LDFLAGS" le_libdir="$ac_cv_libevent_dir" fi if test -d "$ac_cv_libevent_dir/include" ; then CPPFLAGS="-I$ac_cv_libevent_dir/include $CPPFLAGS" else CPPFLAGS="-I$ac_cv_libevent_dir $CPPFLAGS" fi fi trylibssldir="" # Check whether --with-libssl was given. if test "${with_libssl+set}" = set; then : withval=$with_libssl; if test "x$withval" != "xno" ; then trylibssldir=$withval fi fi OPENSSL_URL=https://www.openssl.org/ if test "x$enable_tls" = "xyes"; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for libssl directory" >&5 $as_echo_n "checking for libssl directory... " >&6; } if ${ac_cv_libssl_dir+:} false; then : $as_echo_n "(cached) " >&6 else saved_LIBS="$LIBS" saved_LDFLAGS="$LDFLAGS" saved_CPPFLAGS="$CPPFLAGS" le_found=no for ledir in $trylibssldir "" $prefix /usr/local ; do LDFLAGS="$saved_LDFLAGS" LIBS="-lssl -lcrypto $saved_LIBS" # Skip the directory if it isn't there. if test ! -z "$ledir" -a ! -d "$ledir" ; then continue; fi if test ! -z "$ledir" ; then if test -d "$ledir/lib" ; then LDFLAGS="-L$ledir/lib $LDFLAGS" else LDFLAGS="-L$ledir $LDFLAGS" fi if test -d "$ledir/include" ; then CPPFLAGS="-I$ledir/include $CPPFLAGS" else CPPFLAGS="-I$ledir $CPPFLAGS" fi fi # Can I compile and link it? cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #include #include int main () { SSL_CTX* ssl_ctx = SSL_CTX_new(TLS_server_method()); assert(OPENSSL_VERSION_NUMBER >= 0x10100000L); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : libssl_linked=yes else libssl_linked=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext if test $libssl_linked = yes; then if test ! -z "$ledir" ; then ac_cv_libssl_dir=$ledir _myos=`echo $target_os | cut -f 1 -d .` if test "$SUNCC" = "yes" -o "x$_myos" = "xsolaris2"; then : saved_LDFLAGS="$saved_LDFLAGS -Wl,-R$ledir/lib" else if test "$GCC" = "yes"; then : saved_LDFLAGS="$saved_LDFLAGS -Wl,-rpath,$ledir/lib" fi fi else ac_cv_libssl_dir="(system)" fi le_found=yes break fi done LIBS="$saved_LIBS" LDFLAGS="$saved_LDFLAGS" CPPFLAGS="$saved_CPPFLAGS" if test $le_found = no ; then as_fn_error $? "libssl (at least version 1.1.0) is required. You can get it from $OPENSSL_URL If it's already installed, specify its path using --with-libssl=/dir/ " "$LINENO" 5 fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libssl_dir" >&5 $as_echo "$ac_cv_libssl_dir" >&6; } LIBS="-lssl -lcrypto $LIBS" if test $ac_cv_libssl_dir != "(system)"; then if test -d "$ac_cv_libssl_dir/lib" ; then LDFLAGS="-L$ac_cv_libssl_dir/lib $LDFLAGS" le_libdir="$ac_cv_libssl_dir/lib" else LDFLAGS="-L$ac_cv_libssl_dir $LDFLAGS" le_libdir="$ac_cv_libssl_dir" fi if test -d "$ac_cv_libssl_dir/include" ; then CPPFLAGS="-I$ac_cv_libssl_dir/include $CPPFLAGS" else CPPFLAGS="-I$ac_cv_libssl_dir $CPPFLAGS" fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing umem_cache_create" >&5 $as_echo_n "checking for library containing umem_cache_create... " >&6; } if ${ac_cv_search_umem_cache_create+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char umem_cache_create (); int main () { return umem_cache_create (); ; return 0; } _ACEOF for ac_lib in '' umem; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_umem_cache_create=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_umem_cache_create+:} false; then : break fi done if ${ac_cv_search_umem_cache_create+:} false; then : else ac_cv_search_umem_cache_create=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_umem_cache_create" >&5 $as_echo "$ac_cv_search_umem_cache_create" >&6; } ac_res=$ac_cv_search_umem_cache_create if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing gethugepagesizes" >&5 $as_echo_n "checking for library containing gethugepagesizes... " >&6; } if ${ac_cv_search_gethugepagesizes+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char gethugepagesizes (); int main () { return gethugepagesizes (); ; return 0; } _ACEOF for ac_lib in '' hugetlbfs; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_gethugepagesizes=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_gethugepagesizes+:} false; then : break fi done if ${ac_cv_search_gethugepagesizes+:} false; then : else ac_cv_search_gethugepagesizes=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_gethugepagesizes" >&5 $as_echo "$ac_cv_search_gethugepagesizes" >&6; } ac_res=$ac_cv_search_gethugepagesizes if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdbool.h that conforms to C99" >&5 $as_echo_n "checking for stdbool.h that conforms to C99... " >&6; } if ${ac_cv_header_stdbool_h+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #ifndef bool "error: bool is not defined" #endif #ifndef false "error: false is not defined" #endif #if false "error: false is not 0" #endif #ifndef true "error: true is not defined" #endif #if true != 1 "error: true is not 1" #endif #ifndef __bool_true_false_are_defined "error: __bool_true_false_are_defined is not defined" #endif struct s { _Bool s: 1; _Bool t; } s; char a[true == 1 ? 1 : -1]; char b[false == 0 ? 1 : -1]; char c[__bool_true_false_are_defined == 1 ? 1 : -1]; char d[(bool) 0.5 == true ? 1 : -1]; /* See body of main program for 'e'. */ char f[(_Bool) 0.0 == false ? 1 : -1]; char g[true]; char h[sizeof (_Bool)]; char i[sizeof s.t]; enum { j = false, k = true, l = false * true, m = true * 256 }; /* The following fails for HP aC++/ANSI C B3910B A.05.55 [Dec 04 2003]. */ _Bool n[m]; char o[sizeof n == m * sizeof n[0] ? 1 : -1]; char p[-1 - (_Bool) 0 < 0 && -1 - (bool) 0 < 0 ? 1 : -1]; /* Catch a bug in an HP-UX C compiler. See http://gcc.gnu.org/ml/gcc-patches/2003-12/msg02303.html http://lists.gnu.org/archive/html/bug-coreutils/2005-11/msg00161.html */ _Bool q = true; _Bool *pq = &q; int main () { bool e = &s; *pq |= q; *pq |= ! q; /* Refer to every declared value, to avoid compiler optimizations. */ return (!a + !b + !c + !d + !e + !f + !g + !h + !i + !!j + !k + !!l + !m + !n + !o + !p + !q + !pq); ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_header_stdbool_h=yes else ac_cv_header_stdbool_h=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdbool_h" >&5 $as_echo "$ac_cv_header_stdbool_h" >&6; } ac_fn_c_check_type "$LINENO" "_Bool" "ac_cv_type__Bool" "$ac_includes_default" if test "x$ac_cv_type__Bool" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE__BOOL 1 _ACEOF fi if test $ac_cv_header_stdbool_h = yes; then $as_echo "#define HAVE_STDBOOL_H 1" >>confdefs.h fi for ac_header in inttypes.h do : ac_fn_c_check_header_mongrel "$LINENO" "inttypes.h" "ac_cv_header_inttypes_h" "$ac_includes_default" if test "x$ac_cv_header_inttypes_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_INTTYPES_H 1 _ACEOF fi done { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sasl_callback_ft" >&5 $as_echo_n "checking for sasl_callback_ft... " >&6; } if ${ac_cv_has_sasl_callback_ft+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef HAVE_SASL_SASL_H #include #include #endif int main () { sasl_callback_ft a_callback; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_has_sasl_callback_ft=yes else ac_cv_has_sasl_callback_ft=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_has_sasl_callback_ft" >&5 $as_echo "$ac_cv_has_sasl_callback_ft" >&6; } if test $ac_cv_has_sasl_callback_ft = yes; then $as_echo "#define HAVE_SASL_CALLBACK_FT 1" >>confdefs.h fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for print macros for integers (C99 section 7.8.1)" >&5 $as_echo_n "checking for print macros for integers (C99 section 7.8.1)... " >&6; } if ${ac_cv_c_uint64_support+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef HAVE_INTTYPES_H #include #endif #include int main () { uint64_t val = 0; fprintf(stderr, "%" PRIu64 "\n", val); ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_c_uint64_support=yes else ac_cv_c_uint64_support=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_uint64_support" >&5 $as_echo "$ac_cv_c_uint64_support" >&6; } if test "x$ac_cv_c_uint64_support" = "xno"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Failed to use print macros (PRIu) as defined in C99 section 7.8.1. " >&5 $as_echo "$as_me: WARNING: Failed to use print macros (PRIu) as defined in C99 section 7.8.1. " >&2;} fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for an ANSI C-conforming const" >&5 $as_echo_n "checking for an ANSI C-conforming const... " >&6; } if ${ac_cv_c_const+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { #ifndef __cplusplus /* Ultrix mips cc rejects this sort of thing. */ typedef int charset[2]; const charset cs = { 0, 0 }; /* SunOS 4.1.1 cc rejects this. */ char const *const *pcpcc; char **ppc; /* NEC SVR4.0.2 mips cc rejects this. */ struct point {int x, y;}; static struct point const zero = {0,0}; /* AIX XL C 1.02.0.0 rejects this. It does not let you subtract one const X* pointer from another in an arm of an if-expression whose if-part is not a constant expression */ const char *g = "string"; pcpcc = &g + (g ? g-g : 0); /* HPUX 7.0 cc rejects these. */ ++pcpcc; ppc = (char**) pcpcc; pcpcc = (char const *const *) ppc; { /* SCO 3.2v4 cc rejects this sort of thing. */ char tx; char *t = &tx; char const *s = 0 ? (char *) 0 : (char const *) 0; *t++ = 0; if (s) return 0; } { /* Someone thinks the Sun supposedly-ANSI compiler will reject this. */ int x[] = {25, 17}; const int *foo = &x[0]; ++foo; } { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */ typedef const int *iptr; iptr p = 0; ++p; } { /* AIX XL C 1.02.0.0 rejects this sort of thing, saying "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */ struct s { int j; const int *ap[3]; } bx; struct s *b = &bx; b->j = 5; } { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */ const int foo = 10; if (!foo) return 0; } return !cs[0] && !zero.x; #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_c_const=yes else ac_cv_c_const=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_const" >&5 $as_echo "$ac_cv_c_const" >&6; } if test $ac_cv_c_const = no; then $as_echo "#define const /**/" >>confdefs.h fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for socklen_t" >&5 $as_echo_n "checking for socklen_t... " >&6; } if ${ac_cv_c_socklen_t+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include int main () { socklen_t foo; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_c_socklen_t=yes else ac_cv_c_socklen_t=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_socklen_t" >&5 $as_echo "$ac_cv_c_socklen_t" >&6; } if test $ac_cv_c_socklen_t = no; then $as_echo "#define socklen_t int" >>confdefs.h fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether byte ordering is bigendian" >&5 $as_echo_n "checking whether byte ordering is bigendian... " >&6; } if ${ac_cv_c_bigendian+:} false; then : $as_echo_n "(cached) " >&6 else ac_cv_c_bigendian=unknown # See if we're dealing with a universal compiler. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifndef __APPLE_CC__ not a universal capable compiler #endif typedef int dummy; _ACEOF if ac_fn_c_try_compile "$LINENO"; then : # Check for potential -arch flags. It is not universal unless # there are at least two -arch flags with different values. ac_arch= ac_prev= for ac_word in $CC $CFLAGS $CPPFLAGS $LDFLAGS; do if test -n "$ac_prev"; then case $ac_word in i?86 | x86_64 | ppc | ppc64) if test -z "$ac_arch" || test "$ac_arch" = "$ac_word"; then ac_arch=$ac_word else ac_cv_c_bigendian=universal break fi ;; esac ac_prev= elif test "x$ac_word" = "x-arch"; then ac_prev=arch fi done fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext if test $ac_cv_c_bigendian = unknown; then # See if sys/param.h defines the BYTE_ORDER macro. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include int main () { #if ! (defined BYTE_ORDER && defined BIG_ENDIAN \ && defined LITTLE_ENDIAN && BYTE_ORDER && BIG_ENDIAN \ && LITTLE_ENDIAN) bogus endian macros #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : # It does; now see whether it defined to BIG_ENDIAN or not. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include int main () { #if BYTE_ORDER != BIG_ENDIAN not big endian #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_c_bigendian=yes else ac_cv_c_bigendian=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi if test $ac_cv_c_bigendian = unknown; then # See if defines _LITTLE_ENDIAN or _BIG_ENDIAN (e.g., Solaris). cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { #if ! (defined _LITTLE_ENDIAN || defined _BIG_ENDIAN) bogus endian macros #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : # It does; now see whether it defined to _BIG_ENDIAN or not. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { #ifndef _BIG_ENDIAN not big endian #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_c_bigendian=yes else ac_cv_c_bigendian=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi if test $ac_cv_c_bigendian = unknown; then # Compile a test program. if test "$cross_compiling" = yes; then : # Try to guess by grepping values from an object file. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ short int ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 }; short int ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 }; int use_ascii (int i) { return ascii_mm[i] + ascii_ii[i]; } short int ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 }; short int ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 }; int use_ebcdic (int i) { return ebcdic_mm[i] + ebcdic_ii[i]; } extern int foo; int main () { return use_ascii (foo) == use_ebcdic (foo); ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : if grep BIGenDianSyS conftest.$ac_objext >/dev/null; then ac_cv_c_bigendian=yes fi if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then if test "$ac_cv_c_bigendian" = unknown; then ac_cv_c_bigendian=no else # finding both strings is unlikely to happen, but who knows? ac_cv_c_bigendian=unknown fi fi fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $ac_includes_default int main () { /* Are we little or big endian? From Harbison&Steele. */ union { long int l; char c[sizeof (long int)]; } u; u.l = 1; return u.c[sizeof (long int) - 1] == 1; ; return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : ac_cv_c_bigendian=no else ac_cv_c_bigendian=yes fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_bigendian" >&5 $as_echo "$ac_cv_c_bigendian" >&6; } case $ac_cv_c_bigendian in #( yes) $as_echo "#define ENDIAN_BIG 1" >>confdefs.h ;; #( no) $as_echo "#define ENDIAN_LITTLE 1" >>confdefs.h ;; #( universal) $as_echo "#define AC_APPLE_UNIVERSAL_BUILD 1" >>confdefs.h ;; #( *) as_fn_error $? "Cannot detect endianness. Must pass ac_cv_c_bigendian={yes,no} to configure." "$LINENO" 5 ;; esac { $as_echo "$as_me:${as_lineno-$LINENO}: checking for htonll" >&5 $as_echo_n "checking for htonll... " >&6; } have_htoll="no" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #ifdef HAVE_INTTYPES_H #include */ #endif int main () { return htonll(0); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : have_htoll="yes" $as_echo "#define HAVE_HTONLL 1" >>confdefs.h else have_htoll="no" fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $have_htoll" >&5 $as_echo "$have_htoll" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5 $as_echo_n "checking for library containing pthread_create... " >&6; } if ${ac_cv_search_pthread_create+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char pthread_create (); int main () { return pthread_create (); ; return 0; } _ACEOF for ac_lib in '' pthread; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_pthread_create=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_pthread_create+:} false; then : break fi done if ${ac_cv_search_pthread_create+:} false; then : else ac_cv_search_pthread_create=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 $as_echo "$ac_cv_search_pthread_create" >&6; } ac_res=$ac_cv_search_pthread_create if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi if test "x$ac_cv_search_pthread_create" = "xno"; then as_fn_error $? "Can't enable threads without the POSIX thread library." "$LINENO" 5 fi for ac_func in mlockall do : ac_fn_c_check_func "$LINENO" "mlockall" "ac_cv_func_mlockall" if test "x$ac_cv_func_mlockall" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_MLOCKALL 1 _ACEOF fi done for ac_func in getpagesizes do : ac_fn_c_check_func "$LINENO" "getpagesizes" "ac_cv_func_getpagesizes" if test "x$ac_cv_func_getpagesizes" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_GETPAGESIZES 1 _ACEOF fi done for ac_func in sysconf do : ac_fn_c_check_func "$LINENO" "sysconf" "ac_cv_func_sysconf" if test "x$ac_cv_func_sysconf" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_SYSCONF 1 _ACEOF fi done for ac_func in memcntl do : ac_fn_c_check_func "$LINENO" "memcntl" "ac_cv_func_memcntl" if test "x$ac_cv_func_memcntl" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_MEMCNTL 1 _ACEOF fi done for ac_func in sigignore do : ac_fn_c_check_func "$LINENO" "sigignore" "ac_cv_func_sigignore" if test "x$ac_cv_func_sigignore" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_SIGIGNORE 1 _ACEOF fi done for ac_func in clock_gettime do : ac_fn_c_check_func "$LINENO" "clock_gettime" "ac_cv_func_clock_gettime" if test "x$ac_cv_func_clock_gettime" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_CLOCK_GETTIME 1 _ACEOF fi done for ac_func in accept4 do : ac_fn_c_check_func "$LINENO" "accept4" "ac_cv_func_accept4" if test "x$ac_cv_func_accept4" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_ACCEPT4 1 _ACEOF $as_echo "#define HAVE_ACCEPT4 1" >>confdefs.h fi done for ac_func in getopt_long do : ac_fn_c_check_func "$LINENO" "getopt_long" "ac_cv_func_getopt_long" if test "x$ac_cv_func_getopt_long" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_GETOPT_LONG 1 _ACEOF $as_echo "#define HAVE_GETOPT_LONG 1" >>confdefs.h fi done { $as_echo "$as_me:${as_lineno-$LINENO}: checking for alignment" >&5 $as_echo_n "checking for alignment... " >&6; } if ${ac_cv_c_alignment+:} false; then : $as_echo_n "(cached) " >&6 else if test "$cross_compiling" = yes; then : ac_cv_c_alignment=maybe else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #pragma GCC optimize ("O0") int main () { char *buf = malloc(32); uint64_t *ptr = (uint64_t*)(buf+2); // catch sigbus, etc. *ptr = 0x1; // catch unaligned word access (ARM cpus) #ifdef ENDIAN_BIG #define ALIGNMENT 0x02030405 #else #define ALIGNMENT 0x05040302 #endif *(buf + 0) = 1; *(buf + 1) = 2; *(buf + 2) = 3; *(buf + 3) = 4; *(buf + 4) = 5; int* i = (int*)(buf+1); return (ALIGNMENT == *i) ? 0 : 1; ; return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : ac_cv_c_alignment=none else ac_cv_c_alignment=need fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_alignment" >&5 $as_echo "$ac_cv_c_alignment" >&6; } if test $ac_cv_c_alignment = need; then : $as_echo "#define NEED_ALIGN 1" >>confdefs.h fi if test $ac_cv_c_alignment = maybe; then : { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Assuming aligned access is required when cross-compiling" >&5 $as_echo "$as_me: WARNING: Assuming aligned access is required when cross-compiling" >&2;} $as_echo "#define NEED_ALIGN 1" >>confdefs.h fi have_gcc_atomics=no { $as_echo "$as_me:${as_lineno-$LINENO}: checking for GCC atomics" >&5 $as_echo_n "checking for GCC atomics... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { unsigned short a; unsigned short b; b = __sync_add_and_fetch(&a, 1); b = __sync_sub_and_fetch(&a, 2); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : have_gcc_atomics=yes $as_echo "#define HAVE_GCC_ATOMICS 1" >>confdefs.h fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $have_gcc_atomics" >&5 $as_echo "$have_gcc_atomics" >&6; } have_gcc_64atomics=no { $as_echo "$as_me:${as_lineno-$LINENO}: checking for GCC 64bit atomics" >&5 $as_echo_n "checking for GCC 64bit atomics... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { uint64_t a; uint64_t b; b = __sync_add_and_fetch(&a, 1); b = __sync_sub_and_fetch(&a, 2); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : have_gcc_64atomics=yes $as_echo "#define HAVE_GCC_64ATOMICS 1" >>confdefs.h fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $have_gcc_64atomics" >&5 $as_echo "$have_gcc_64atomics" >&6; } for ac_func in setppriv do : ac_fn_c_check_func "$LINENO" "setppriv" "ac_cv_func_setppriv" if test "x$ac_cv_func_setppriv" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_SETPPRIV 1 _ACEOF ac_fn_c_check_header_mongrel "$LINENO" "priv.h" "ac_cv_header_priv_h" "$ac_includes_default" if test "x$ac_cv_header_priv_h" = xyes; then : $as_echo "#define HAVE_DROP_PRIVILEGES 1" >>confdefs.h build_solaris_privs=yes fi fi done if test "x$enable_seccomp" = "xyes" ; then : { $as_echo "$as_me:${as_lineno-$LINENO}: checking for seccomp_rule_add in -lseccomp" >&5 $as_echo_n "checking for seccomp_rule_add in -lseccomp... " >&6; } if ${ac_cv_lib_seccomp_seccomp_rule_add+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS LIBS="-lseccomp $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char seccomp_rule_add (); int main () { return seccomp_rule_add (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : ac_cv_lib_seccomp_seccomp_rule_add=yes else ac_cv_lib_seccomp_seccomp_rule_add=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_seccomp_seccomp_rule_add" >&5 $as_echo "$ac_cv_lib_seccomp_seccomp_rule_add" >&6; } if test "x$ac_cv_lib_seccomp_seccomp_rule_add" = xyes; then : $as_echo "#define HAVE_DROP_PRIVILEGES 1" >>confdefs.h build_linux_privs=yes $as_echo "#define HAVE_DROP_WORKER_PRIVILEGES 1" >>confdefs.h build_linux_privs=yes fi fi for ac_func in pledge do : ac_fn_c_check_func "$LINENO" "pledge" "ac_cv_func_pledge" if test "x$ac_cv_func_pledge" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_PLEDGE 1 _ACEOF ac_fn_c_check_header_mongrel "$LINENO" "unistd.h" "ac_cv_header_unistd_h" "$ac_includes_default" if test "x$ac_cv_header_unistd_h" = xyes; then : $as_echo "#define HAVE_DROP_PRIVILEGES 1" >>confdefs.h build_openbsd_privs=yes fi fi done for ac_func in cap_enter do : ac_fn_c_check_func "$LINENO" "cap_enter" "ac_cv_func_cap_enter" if test "x$ac_cv_func_cap_enter" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_CAP_ENTER 1 _ACEOF ac_fn_c_check_header_mongrel "$LINENO" "sys/capsicum.h" "ac_cv_header_sys_capsicum_h" "$ac_includes_default" if test "x$ac_cv_header_sys_capsicum_h" = xyes; then : $as_echo "#define HAVE_DROP_PRIVILEGES 1" >>confdefs.h build_freebsd_privs=yes fi fi done if test "$build_solaris_privs" = "yes"; then BUILD_SOLARIS_PRIVS_TRUE= BUILD_SOLARIS_PRIVS_FALSE='#' else BUILD_SOLARIS_PRIVS_TRUE='#' BUILD_SOLARIS_PRIVS_FALSE= fi if test "$build_linux_privs" = "yes"; then BUILD_LINUX_PRIVS_TRUE= BUILD_LINUX_PRIVS_FALSE='#' else BUILD_LINUX_PRIVS_TRUE='#' BUILD_LINUX_PRIVS_FALSE= fi if test "$build_openbsd_privs" = "yes"; then BUILD_OPENBSD_PRIVS_TRUE= BUILD_OPENBSD_PRIVS_FALSE='#' else BUILD_OPENBSD_PRIVS_TRUE='#' BUILD_OPENBSD_PRIVS_FALSE= fi if test "$build_freebsd_privs" = "yes"; then BUILD_FREEBSD_PRIVS_TRUE= BUILD_FREEBSD_PRIVS_FALSE='#' else BUILD_FREEBSD_PRIVS_TRUE='#' BUILD_FREEBSD_PRIVS_FALSE= fi ac_fn_c_check_header_mongrel "$LINENO" "umem.h" "ac_cv_header_umem_h" "$ac_includes_default" if test "x$ac_cv_header_umem_h" = xyes; then : $as_echo "#define HAVE_UMEM_H 1" >>confdefs.h build_cache=no else build_cache=yes fi if test "x$build_cache" = "xyes"; then BUILD_CACHE_TRUE= BUILD_CACHE_FALSE='#' else BUILD_CACHE_TRUE='#' BUILD_CACHE_FALSE= fi # Check whether --enable-docs was given. if test "${enable_docs+set}" = set; then : enableval=$enable_docs; fi # Extract the first word of "xml2rfc", so it can be a program name with args. set dummy xml2rfc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_XML2RFC+:} false; then : $as_echo_n "(cached) " >&6 else case $XML2RFC in [\\/]* | ?:[\\/]*) ac_cv_path_XML2RFC="$XML2RFC" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_XML2RFC="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS test -z "$ac_cv_path_XML2RFC" && ac_cv_path_XML2RFC=""no"" ;; esac fi XML2RFC=$ac_cv_path_XML2RFC if test -n "$XML2RFC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $XML2RFC" >&5 $as_echo "$XML2RFC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi # Extract the first word of "xsltproc", so it can be a program name with args. set dummy xsltproc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_XSLTPROC+:} false; then : $as_echo_n "(cached) " >&6 else case $XSLTPROC in [\\/]* | ?:[\\/]*) ac_cv_path_XSLTPROC="$XSLTPROC" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_XSLTPROC="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS test -z "$ac_cv_path_XSLTPROC" && ac_cv_path_XSLTPROC=""no"" ;; esac fi XSLTPROC=$ac_cv_path_XSLTPROC if test -n "$XSLTPROC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $XSLTPROC" >&5 $as_echo "$XSLTPROC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi if test "x$enable_docs" != "xno" -a "x$XML2RFC" != "xno" -a "x$XSLTPROC" != "xno"; then BUILD_SPECIFICATIONS_TRUE= BUILD_SPECIFICATIONS_FALSE='#' else BUILD_SPECIFICATIONS_TRUE='#' BUILD_SPECIFICATIONS_FALSE= fi if test "$ICC" = "yes" then CFLAGS="$CFLAGS -diag-disable 187 -Wall -Werror" $as_echo "#define _GNU_SOURCE 1" >>confdefs.h elif test "$GCC" = "yes" then GCC_VERSION=`$CC -dumpversion` CFLAGS="$CFLAGS -Wall -Werror -pedantic -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls" if test "x$enable_asan" = "xyes"; then CFLAGS="$CFLAGS -fsanitize=address" fi case $GCC_VERSION in 4.4.*) CFLAGS="$CFLAGS -fno-strict-aliasing" ;; esac $as_echo "#define _GNU_SOURCE 1" >>confdefs.h elif test "$SUNCC" = "yes" then CFLAGS="$CFLAGS -errfmt=error -errwarn -errshort=tags" fi ac_config_files="$ac_config_files Makefile doc/Makefile" cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure # tests run on this system so they can be shared between configure # scripts and configure runs, see configure's option --config-cache. # It is not useful on other systems. If it contains results you don't # want to keep, you may remove or edit it. # # config.status only pays attention to the cache file if you give it # the --recheck option to rerun configure. # # `ac_cv_env_foo' variables (set or unset) will be overridden when # loading this file, other *unset* `ac_cv_foo' will be assigned the # following values. _ACEOF # The following way of writing the cache mishandles newlines in values, # but we know of no workaround that is simple, portable, and efficient. # So, we kill variables containing newlines. # Ultrix sh set writes to stderr and can't be redirected directly, # and sets the high bit in the cache file unless we assign to the vars. ( for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do eval ac_val=\$$ac_var case $ac_val in #( *${as_nl}*) case $ac_var in #( *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; esac case $ac_var in #( _ | IFS | as_nl) ;; #( BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( *) { eval $ac_var=; unset $ac_var;} ;; esac ;; esac done (set) 2>&1 | case $as_nl`(ac_space=' '; set) 2>&1` in #( *${as_nl}ac_space=\ *) # `set' does not quote correctly, so add quotes: double-quote # substitution turns \\\\ into \\, and sed turns \\ into \. sed -n \ "s/'/'\\\\''/g; s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" ;; #( *) # `set' quotes correctly as required by POSIX, so do not add quotes. sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" ;; esac | sort ) | sed ' /^ac_cv_env_/b end t clear :clear s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/ t end s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/ :end' >>confcache if diff "$cache_file" confcache >/dev/null 2>&1; then :; else if test -w "$cache_file"; then if test "x$cache_file" != "x/dev/null"; then { $as_echo "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 $as_echo "$as_me: updating cache $cache_file" >&6;} if test ! -f "$cache_file" || test -h "$cache_file"; then cat confcache >"$cache_file" else case $cache_file in #( */* | ?:*) mv -f confcache "$cache_file"$$ && mv -f "$cache_file"$$ "$cache_file" ;; #( *) mv -f confcache "$cache_file" ;; esac fi fi else { $as_echo "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 $as_echo "$as_me: not updating unwritable cache $cache_file" >&6;} fi fi rm -f confcache test "x$prefix" = xNONE && prefix=$ac_default_prefix # Let make expand exec_prefix. test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' DEFS=-DHAVE_CONFIG_H ac_libobjs= ac_ltlibobjs= for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue # 1. Remove the extension, and $U if already installed. ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' ac_i=`$as_echo "$ac_i" | sed "$ac_script"` # 2. Prepend LIBOBJDIR. When used with automake>=1.10 LIBOBJDIR # will be set to the directory where LIBOBJS objects are built. as_fn_append ac_libobjs " \${LIBOBJDIR}$ac_i\$U.$ac_objext" as_fn_append ac_ltlibobjs " \${LIBOBJDIR}$ac_i"'$U.lo' done LIBOBJS=$ac_libobjs LTLIBOBJS=$ac_ltlibobjs { $as_echo "$as_me:${as_lineno-$LINENO}: checking that generated files are newer than configure" >&5 $as_echo_n "checking that generated files are newer than configure... " >&6; } if test -n "$am_sleep_pid"; then # Hide warnings about reused PIDs. wait $am_sleep_pid 2>/dev/null fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: done" >&5 $as_echo "done" >&6; } if test -n "$EXEEXT"; then am__EXEEXT_TRUE= am__EXEEXT_FALSE='#' else am__EXEEXT_TRUE='#' am__EXEEXT_FALSE= fi if test -z "${AMDEP_TRUE}" && test -z "${AMDEP_FALSE}"; then as_fn_error $? "conditional \"AMDEP\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${am__fastdepCC_TRUE}" && test -z "${am__fastdepCC_FALSE}"; then as_fn_error $? "conditional \"am__fastdepCC\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${BUILD_DTRACE_TRUE}" && test -z "${BUILD_DTRACE_FALSE}"; then as_fn_error $? "conditional \"BUILD_DTRACE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${DTRACE_INSTRUMENT_OBJ_TRUE}" && test -z "${DTRACE_INSTRUMENT_OBJ_FALSE}"; then as_fn_error $? "conditional \"DTRACE_INSTRUMENT_OBJ\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${ENABLE_SASL_TRUE}" && test -z "${ENABLE_SASL_FALSE}"; then as_fn_error $? "conditional \"ENABLE_SASL\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${ENABLE_EXTSTORE_TRUE}" && test -z "${ENABLE_EXTSTORE_FALSE}"; then as_fn_error $? "conditional \"ENABLE_EXTSTORE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${ENABLE_ARM_CRC32_TRUE}" && test -z "${ENABLE_ARM_CRC32_FALSE}"; then as_fn_error $? "conditional \"ENABLE_ARM_CRC32\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${ENABLE_TLS_TRUE}" && test -z "${ENABLE_TLS_FALSE}"; then as_fn_error $? "conditional \"ENABLE_TLS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${ENABLE_ASAN_TRUE}" && test -z "${ENABLE_ASAN_FALSE}"; then as_fn_error $? "conditional \"ENABLE_ASAN\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${BUILD_SOLARIS_PRIVS_TRUE}" && test -z "${BUILD_SOLARIS_PRIVS_FALSE}"; then as_fn_error $? "conditional \"BUILD_SOLARIS_PRIVS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${BUILD_LINUX_PRIVS_TRUE}" && test -z "${BUILD_LINUX_PRIVS_FALSE}"; then as_fn_error $? "conditional \"BUILD_LINUX_PRIVS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${BUILD_OPENBSD_PRIVS_TRUE}" && test -z "${BUILD_OPENBSD_PRIVS_FALSE}"; then as_fn_error $? "conditional \"BUILD_OPENBSD_PRIVS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${BUILD_FREEBSD_PRIVS_TRUE}" && test -z "${BUILD_FREEBSD_PRIVS_FALSE}"; then as_fn_error $? "conditional \"BUILD_FREEBSD_PRIVS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${BUILD_CACHE_TRUE}" && test -z "${BUILD_CACHE_FALSE}"; then as_fn_error $? "conditional \"BUILD_CACHE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${BUILD_SPECIFICATIONS_TRUE}" && test -z "${BUILD_SPECIFICATIONS_FALSE}"; then as_fn_error $? "conditional \"BUILD_SPECIFICATIONS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 ac_clean_files_save=$ac_clean_files ac_clean_files="$ac_clean_files $CONFIG_STATUS" { $as_echo "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5 $as_echo "$as_me: creating $CONFIG_STATUS" >&6;} as_write_fail=0 cat >$CONFIG_STATUS <<_ASEOF || as_write_fail=1 #! $SHELL # Generated by $as_me. # Run this file to recreate the current configuration. # Compiler output produced by configure, useful for debugging # configure, is in config.log if it exists. debug=false ac_cs_recheck=false ac_cs_silent=false SHELL=\${CONFIG_SHELL-$SHELL} export SHELL _ASEOF cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1 ## -------------------- ## ## M4sh Initialization. ## ## -------------------- ## # Be more Bourne compatible DUALCASE=1; export DUALCASE # for MKS sh if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which # is contrary to our usage. Disable this feature. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST else case `(set -o) 2>/dev/null` in #( *posix*) : set -o posix ;; #( *) : ;; esac fi as_nl=' ' export as_nl # Printing a long string crashes Solaris 7 /usr/bin/printf. as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo # Prefer a ksh shell builtin over an external printf program on Solaris, # but without wasting forks for bash or zsh. if test -z "$BASH_VERSION$ZSH_VERSION" \ && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then as_echo='print -r --' as_echo_n='print -rn --' elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then as_echo='printf %s\n' as_echo_n='printf %s' else if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' as_echo_n='/usr/ucb/echo -n' else as_echo_body='eval expr "X$1" : "X\\(.*\\)"' as_echo_n_body='eval arg=$1; case $arg in #( *"$as_nl"*) expr "X$arg" : "X\\(.*\\)$as_nl"; arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; esac; expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" ' export as_echo_n_body as_echo_n='sh -c $as_echo_n_body as_echo' fi export as_echo_body as_echo='sh -c $as_echo_body as_echo' fi # The user is always right. if test "${PATH_SEPARATOR+set}" != set; then PATH_SEPARATOR=: (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || PATH_SEPARATOR=';' } fi # IFS # We need space, tab and new line, in precisely that order. Quoting is # there to prevent editors from complaining about space-tab. # (If _AS_PATH_WALK were called with IFS unset, it would disable word # splitting by setting IFS to empty value.) IFS=" "" $as_nl" # Find who we are. Look in the path if we contain no directory separator. as_myself= case $0 in #(( *[\\/]* ) as_myself=$0 ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break done IFS=$as_save_IFS ;; esac # We did not find ourselves, most probably we were run as `sh COMMAND' # in which case we are not to be found in the path. if test "x$as_myself" = x; then as_myself=$0 fi if test ! -f "$as_myself"; then $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 exit 1 fi # Unset variables that we do not need and which cause bugs (e.g. in # pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" # suppresses any "Segmentation fault" message there. '((' could # trigger a bug in pdksh 5.2.14. for as_var in BASH_ENV ENV MAIL MAILPATH do eval test x\${$as_var+set} = xset \ && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : done PS1='$ ' PS2='> ' PS4='+ ' # NLS nuisances. LC_ALL=C export LC_ALL LANGUAGE=C export LANGUAGE # CDPATH. (unset CDPATH) >/dev/null 2>&1 && unset CDPATH # as_fn_error STATUS ERROR [LINENO LOG_FD] # ---------------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the # script with STATUS, using 1 if that was 0. as_fn_error () { as_status=$1; test $as_status -eq 0 && as_status=1 if test "$4"; then as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi $as_echo "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error # as_fn_set_status STATUS # ----------------------- # Set $? to STATUS, without forking. as_fn_set_status () { return $1 } # as_fn_set_status # as_fn_exit STATUS # ----------------- # Exit the shell with STATUS, even in a "trap 0" or "set -e" context. as_fn_exit () { set +e as_fn_set_status $1 exit $1 } # as_fn_exit # as_fn_unset VAR # --------------- # Portably unset VAR. as_fn_unset () { { eval $1=; unset $1;} } as_unset=as_fn_unset # as_fn_append VAR VALUE # ---------------------- # Append the text in VALUE to the end of the definition contained in VAR. Take # advantage of any shell optimizations that allow amortized linear growth over # repeated appends, instead of the typical quadratic growth present in naive # implementations. if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : eval 'as_fn_append () { eval $1+=\$2 }' else as_fn_append () { eval $1=\$$1\$2 } fi # as_fn_append # as_fn_arith ARG... # ------------------ # Perform arithmetic evaluation on the ARGs, and store the result in the # global $as_val. Take advantage of shells that can avoid forks. The arguments # must be portable across $(()) and expr. if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : eval 'as_fn_arith () { as_val=$(( $* )) }' else as_fn_arith () { as_val=`expr "$@" || test $? -eq 1` } fi # as_fn_arith if expr a : '\(a\)' >/dev/null 2>&1 && test "X`expr 00001 : '.*\(...\)'`" = X001; then as_expr=expr else as_expr=false fi if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then as_basename=basename else as_basename=false fi if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then as_dirname=dirname else as_dirname=false fi as_me=`$as_basename -- "$0" || $as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ X"$0" : 'X\(//\)$' \| \ X"$0" : 'X\(/\)' \| . 2>/dev/null || $as_echo X/"$0" | sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/ q } /^X\/\(\/\/\)$/{ s//\1/ q } /^X\/\(\/\).*/{ s//\1/ q } s/.*/./; q'` # Avoid depending upon Character Ranges. as_cr_letters='abcdefghijklmnopqrstuvwxyz' as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' as_cr_Letters=$as_cr_letters$as_cr_LETTERS as_cr_digits='0123456789' as_cr_alnum=$as_cr_Letters$as_cr_digits ECHO_C= ECHO_N= ECHO_T= case `echo -n x` in #((((( -n*) case `echo 'xy\c'` in *c*) ECHO_T=' ';; # ECHO_T is single tab character. xy) ECHO_C='\c';; *) echo `echo ksh88 bug on AIX 6.1` > /dev/null ECHO_T=' ';; esac;; *) ECHO_N='-n';; esac rm -f conf$$ conf$$.exe conf$$.file if test -d conf$$.dir; then rm -f conf$$.dir/conf$$.file else rm -f conf$$.dir mkdir conf$$.dir 2>/dev/null fi if (echo >conf$$.file) 2>/dev/null; then if ln -s conf$$.file conf$$ 2>/dev/null; then as_ln_s='ln -s' # ... but there are two gotchas: # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. # In both cases, we have to default to `cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then as_ln_s=ln else as_ln_s='cp -pR' fi else as_ln_s='cp -pR' fi rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file rmdir conf$$.dir 2>/dev/null # as_fn_mkdir_p # ------------- # Create "$as_dir" as a directory, including parents if necessary. as_fn_mkdir_p () { case $as_dir in #( -*) as_dir=./$as_dir;; esac test -d "$as_dir" || eval $as_mkdir_p || { as_dirs= while :; do case $as_dir in #( *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( *) as_qdir=$as_dir;; esac as_dirs="'$as_qdir' $as_dirs" as_dir=`$as_dirname -- "$as_dir" || $as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_dir" : 'X\(//\)[^/]' \| \ X"$as_dir" : 'X\(//\)$' \| \ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$as_dir" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" } # as_fn_mkdir_p if mkdir -p . 2>/dev/null; then as_mkdir_p='mkdir -p "$as_dir"' else test -d ./-p && rmdir ./-p as_mkdir_p=false fi # as_fn_executable_p FILE # ----------------------- # Test if FILE is an executable regular file. as_fn_executable_p () { test -f "$1" && test -x "$1" } # as_fn_executable_p as_test_x='test -x' as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" # Sed expression to map a string onto a valid variable name. as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" exec 6>&1 ## ----------------------------------- ## ## Main body of $CONFIG_STATUS script. ## ## ----------------------------------- ## _ASEOF test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" This file was extended by memcached $as_me 1.5.22, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS CONFIG_COMMANDS = $CONFIG_COMMANDS $ $0 $@ on `(hostname || uname -n) 2>/dev/null | sed 1q` " _ACEOF case $ac_config_files in *" "*) set x $ac_config_files; shift; ac_config_files=$*;; esac case $ac_config_headers in *" "*) set x $ac_config_headers; shift; ac_config_headers=$*;; esac cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 # Files that config.status was made for. config_files="$ac_config_files" config_headers="$ac_config_headers" config_commands="$ac_config_commands" _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 ac_cs_usage="\ \`$as_me' instantiates files and other configuration actions from templates according to the current configuration. Unless the files and actions are specified as TAGs, all are instantiated by default. Usage: $0 [OPTION]... [TAG]... -h, --help print this help, then exit -V, --version print version number and configuration settings, then exit --config print configuration, then exit -q, --quiet, --silent do not print progress messages -d, --debug don't remove temporary files --recheck update $as_me by reconfiguring in the same conditions --file=FILE[:TEMPLATE] instantiate the configuration file FILE --header=FILE[:TEMPLATE] instantiate the configuration header FILE Configuration files: $config_files Configuration headers: $config_headers Configuration commands: $config_commands Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ memcached config.status 1.5.22 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." ac_pwd='$ac_pwd' srcdir='$srcdir' INSTALL='$INSTALL' MKDIR_P='$MKDIR_P' AWK='$AWK' test -n "\$AWK" || AWK=awk _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # The default lists apply if the user does not specify any file. ac_need_defaults=: while test $# != 0 do case $1 in --*=?*) ac_option=`expr "X$1" : 'X\([^=]*\)='` ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` ac_shift=: ;; --*=) ac_option=`expr "X$1" : 'X\([^=]*\)='` ac_optarg= ac_shift=: ;; *) ac_option=$1 ac_optarg=$2 ac_shift=shift ;; esac case $ac_option in # Handling of the options. -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) ac_cs_recheck=: ;; --version | --versio | --versi | --vers | --ver | --ve | --v | -V ) $as_echo "$ac_cs_version"; exit ;; --config | --confi | --conf | --con | --co | --c ) $as_echo "$ac_cs_config"; exit ;; --debug | --debu | --deb | --de | --d | -d ) debug=: ;; --file | --fil | --fi | --f ) $ac_shift case $ac_optarg in *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; '') as_fn_error $? "missing file argument" ;; esac as_fn_append CONFIG_FILES " '$ac_optarg'" ac_need_defaults=false;; --header | --heade | --head | --hea ) $ac_shift case $ac_optarg in *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; esac as_fn_append CONFIG_HEADERS " '$ac_optarg'" ac_need_defaults=false;; --he | --h) # Conflict between --help and --header as_fn_error $? "ambiguous option: \`$1' Try \`$0 --help' for more information.";; --help | --hel | -h ) $as_echo "$ac_cs_usage"; exit ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ | -silent | --silent | --silen | --sile | --sil | --si | --s) ac_cs_silent=: ;; # This is an error. -*) as_fn_error $? "unrecognized option: \`$1' Try \`$0 --help' for more information." ;; *) as_fn_append ac_config_targets " $1" ac_need_defaults=false ;; esac shift done ac_configure_extra_args= if $ac_cs_silent; then exec 6>/dev/null ac_configure_extra_args="$ac_configure_extra_args --silent" fi _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 if \$ac_cs_recheck; then set X $SHELL '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion shift \$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6 CONFIG_SHELL='$SHELL' export CONFIG_SHELL exec "\$@" fi _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 exec 5>>config.log { echo sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX ## Running $as_me. ## _ASBOX $as_echo "$ac_log" } >&5 _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 # # INIT-COMMANDS # AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir" _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Handling of arguments. for ac_config_target in $ac_config_targets do case $ac_config_target in "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;; "depfiles") CONFIG_COMMANDS="$CONFIG_COMMANDS depfiles" ;; "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; "doc/Makefile") CONFIG_FILES="$CONFIG_FILES doc/Makefile" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac done # If the user did not use the arguments to specify the items to instantiate, # then the envvar interface is used. Set only those that are not. # We use the long form for the default assignment because of an extremely # bizarre bug on SunOS 4.1.3. if $ac_need_defaults; then test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands fi # Have a temporary directory for convenience. Make it in the build tree # simply because there is no reason against having it here, and in addition, # creating and moving files from /tmp can sometimes cause problems. # Hook for its removal unless debugging. # Note that there is a small window in which the directory will not be cleaned: # after its creation but before its name has been assigned to `$tmp'. $debug || { tmp= ac_tmp= trap 'exit_status=$? : "${ac_tmp:=$tmp}" { test ! -d "$ac_tmp" || rm -fr "$ac_tmp"; } && exit $exit_status ' 0 trap 'as_fn_exit 1' 1 2 13 15 } # Create a (secure) tmp directory for tmp files. { tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` && test -d "$tmp" } || { tmp=./conf$$-$RANDOM (umask 077 && mkdir "$tmp") } || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5 ac_tmp=$tmp # Set up the scripts for CONFIG_FILES section. # No need to generate them if there are no CONFIG_FILES. # This happens for instance with `./config.status config.h'. if test -n "$CONFIG_FILES"; then ac_cr=`echo X | tr X '\015'` # On cygwin, bash can eat \r inside `` if the user requested igncr. # But we know of no other shell where ac_cr would be empty at this # point, so we can use a bashism as a fallback. if test "x$ac_cr" = x; then eval ac_cr=\$\'\\r\' fi ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' /dev/null` if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then ac_cs_awk_cr='\\r' else ac_cs_awk_cr=$ac_cr fi echo 'BEGIN {' >"$ac_tmp/subs1.awk" && _ACEOF { echo "cat >conf$$subs.awk <<_ACEOF" && echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && echo "_ACEOF" } >conf$$subs.sh || as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` ac_delim='%!_!# ' for ac_last_try in false false false false false :; do . ./conf$$subs.sh || as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` if test $ac_delim_n = $ac_delim_num; then break elif $ac_last_try; then as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi done rm -f conf$$subs.sh cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK && _ACEOF sed -n ' h s/^/S["/; s/!.*/"]=/ p g s/^[^!]*!// :repl t repl s/'"$ac_delim"'$// t delim :nl h s/\(.\{148\}\)..*/\1/ t more1 s/["\\]/\\&/g; s/^/"/; s/$/\\n"\\/ p n b repl :more1 s/["\\]/\\&/g; s/^/"/; s/$/"\\/ p g s/.\{148\}// t nl :delim h s/\(.\{148\}\)..*/\1/ t more2 s/["\\]/\\&/g; s/^/"/; s/$/"/ p b :more2 s/["\\]/\\&/g; s/^/"/; s/$/"\\/ p g s/.\{148\}// t delim ' >$CONFIG_STATUS || ac_write_fail=1 rm -f conf$$subs.awk cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 _ACAWK cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && for (key in S) S_is_set[key] = 1 FS = "" } { line = $ 0 nfields = split(line, field, "@") substed = 0 len = length(field[1]) for (i = 2; i < nfields; i++) { key = field[i] keylen = length(key) if (S_is_set[key]) { value = S[key] line = substr(line, 1, len) "" value "" substr(line, len + keylen + 3) len += length(value) + length(field[++i]) substed = 1 } else len += 1 + keylen } print line } _ACAWK _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" else cat fi < "$ac_tmp/subs1.awk" > "$ac_tmp/subs.awk" \ || as_fn_error $? "could not setup config files machinery" "$LINENO" 5 _ACEOF # VPATH may cause trouble with some makes, so we remove sole $(srcdir), # ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and # trailing colons and then remove the whole line if VPATH becomes empty # (actually we leave an empty line to preserve line numbers). if test "x$srcdir" = x.; then ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{ h s/// s/^/:/ s/[ ]*$/:/ s/:\$(srcdir):/:/g s/:\${srcdir}:/:/g s/:@srcdir@:/:/g s/^:*// s/:*$// x s/\(=[ ]*\).*/\1/ G s/\n// s/^[^=]*=[ ]*$// }' fi cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 fi # test -n "$CONFIG_FILES" # Set up the scripts for CONFIG_HEADERS section. # No need to generate them if there are no CONFIG_HEADERS. # This happens for instance with `./config.status Makefile'. if test -n "$CONFIG_HEADERS"; then cat >"$ac_tmp/defines.awk" <<\_ACAWK || BEGIN { _ACEOF # Transform confdefs.h into an awk script `defines.awk', embedded as # here-document in config.status, that substitutes the proper values into # config.h.in to produce config.h. # Create a delimiter string that does not exist in confdefs.h, to ease # handling of long lines. ac_delim='%!_!# ' for ac_last_try in false false :; do ac_tt=`sed -n "/$ac_delim/p" confdefs.h` if test -z "$ac_tt"; then break elif $ac_last_try; then as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi done # For the awk script, D is an array of macro values keyed by name, # likewise P contains macro parameters if any. Preserve backslash # newline sequences. ac_word_re=[_$as_cr_Letters][_$as_cr_alnum]* sed -n ' s/.\{148\}/&'"$ac_delim"'/g t rset :rset s/^[ ]*#[ ]*define[ ][ ]*/ / t def d :def s/\\$// t bsnl s/["\\]/\\&/g s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ D["\1"]=" \3"/p s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2"/p d :bsnl s/["\\]/\\&/g s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ D["\1"]=" \3\\\\\\n"\\/p t cont s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2\\\\\\n"\\/p t cont d :cont n s/.\{148\}/&'"$ac_delim"'/g t clear :clear s/\\$// t bsnlc s/["\\]/\\&/g; s/^/"/; s/$/"/p d :bsnlc s/["\\]/\\&/g; s/^/"/; s/$/\\\\\\n"\\/p b cont ' >$CONFIG_STATUS || ac_write_fail=1 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 for (key in D) D_is_set[key] = 1 FS = "" } /^[\t ]*#[\t ]*(define|undef)[\t ]+$ac_word_re([\t (]|\$)/ { line = \$ 0 split(line, arg, " ") if (arg[1] == "#") { defundef = arg[2] mac1 = arg[3] } else { defundef = substr(arg[1], 2) mac1 = arg[2] } split(mac1, mac2, "(") #) macro = mac2[1] prefix = substr(line, 1, index(line, defundef) - 1) if (D_is_set[macro]) { # Preserve the white space surrounding the "#". print prefix "define", macro P[macro] D[macro] next } else { # Replace #undef with comments. This is necessary, for example, # in the case of _POSIX_SOURCE, which is predefined and required # on some systems where configure will not decide to define it. if (defundef == "undef") { print "/*", prefix defundef, macro, "*/" next } } } { print } _ACAWK _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 as_fn_error $? "could not setup config headers machinery" "$LINENO" 5 fi # test -n "$CONFIG_HEADERS" eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS :C $CONFIG_COMMANDS" shift for ac_tag do case $ac_tag in :[FHLC]) ac_mode=$ac_tag; continue;; esac case $ac_mode$ac_tag in :[FHL]*:*);; :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; :[FH]-) ac_tag=-:-;; :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; esac ac_save_IFS=$IFS IFS=: set x $ac_tag IFS=$ac_save_IFS shift ac_file=$1 shift case $ac_mode in :L) ac_source=$1;; :[FH]) ac_file_inputs= for ac_f do case $ac_f in -) ac_f="$ac_tmp/stdin";; *) # Look for the file first in the build tree, then in the source tree # (if the path is not absolute). The absolute path cannot be DOS-style, # because $ac_f cannot contain `:'. test -f "$ac_f" || case $ac_f in [\\/$]*) false;; *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; esac || as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; esac case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac as_fn_append ac_file_inputs " '$ac_f'" done # Let's still pretend it is `configure' which instantiates (i.e., don't # use $as_me), people would be surprised to read: # /* config.h. Generated by config.status. */ configure_input='Generated from '` $as_echo "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g' `' by configure.' if test x"$ac_file" != x-; then configure_input="$ac_file. $configure_input" { $as_echo "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5 $as_echo "$as_me: creating $ac_file" >&6;} fi # Neutralize special characters interpreted by sed in replacement strings. case $configure_input in #( *\&* | *\|* | *\\* ) ac_sed_conf_input=`$as_echo "$configure_input" | sed 's/[\\\\&|]/\\\\&/g'`;; #( *) ac_sed_conf_input=$configure_input;; esac case $ac_tag in *:-:* | *:-) cat >"$ac_tmp/stdin" \ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; esac ;; esac ac_dir=`$as_dirname -- "$ac_file" || $as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$ac_file" : 'X\(//\)[^/]' \| \ X"$ac_file" : 'X\(//\)$' \| \ X"$ac_file" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$ac_file" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` as_dir="$ac_dir"; as_fn_mkdir_p ac_builddir=. case "$ac_dir" in .) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` # A ".." for each directory in $ac_dir_suffix. ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` case $ac_top_builddir_sub in "") ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; esac ;; esac ac_abs_top_builddir=$ac_pwd ac_abs_builddir=$ac_pwd$ac_dir_suffix # for backward compatibility: ac_top_builddir=$ac_top_build_prefix case $srcdir in .) # We are building in place. ac_srcdir=. ac_top_srcdir=$ac_top_builddir_sub ac_abs_top_srcdir=$ac_pwd ;; [\\/]* | ?:[\\/]* ) # Absolute name. ac_srcdir=$srcdir$ac_dir_suffix; ac_top_srcdir=$srcdir ac_abs_top_srcdir=$srcdir ;; *) # Relative name. ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix ac_top_srcdir=$ac_top_build_prefix$srcdir ac_abs_top_srcdir=$ac_pwd/$srcdir ;; esac ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix case $ac_mode in :F) # # CONFIG_FILE # case $INSTALL in [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;; *) ac_INSTALL=$ac_top_build_prefix$INSTALL ;; esac ac_MKDIR_P=$MKDIR_P case $MKDIR_P in [\\/$]* | ?:[\\/]* ) ;; */*) ac_MKDIR_P=$ac_top_build_prefix$MKDIR_P ;; esac _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # If the template does not know about datarootdir, expand it. # FIXME: This hack should be removed a few years after 2.60. ac_datarootdir_hack=; ac_datarootdir_seen= ac_sed_dataroot=' /datarootdir/ { p q } /@datadir@/p /@docdir@/p /@infodir@/p /@localedir@/p /@mandir@/p' case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in *datarootdir*) ac_datarootdir_seen=yes;; *@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5 $as_echo "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;} _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_datarootdir_hack=' s&@datadir@&$datadir&g s&@docdir@&$docdir&g s&@infodir@&$infodir&g s&@localedir@&$localedir&g s&@mandir@&$mandir&g s&\\\${datarootdir}&$datarootdir&g' ;; esac _ACEOF # Neutralize VPATH when `$srcdir' = `.'. # Shell code in configure.ac might set extrasub. # FIXME: do we really want to maintain this feature? cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_sed_extra="$ac_vpsub $extrasub _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 :t /@[a-zA-Z_][a-zA-Z_0-9]*@/!b s|@configure_input@|$ac_sed_conf_input|;t t s&@top_builddir@&$ac_top_builddir_sub&;t t s&@top_build_prefix@&$ac_top_build_prefix&;t t s&@srcdir@&$ac_srcdir&;t t s&@abs_srcdir@&$ac_abs_srcdir&;t t s&@top_srcdir@&$ac_top_srcdir&;t t s&@abs_top_srcdir@&$ac_abs_top_srcdir&;t t s&@builddir@&$ac_builddir&;t t s&@abs_builddir@&$ac_abs_builddir&;t t s&@abs_top_builddir@&$ac_abs_top_builddir&;t t s&@INSTALL@&$ac_INSTALL&;t t s&@MKDIR_P@&$ac_MKDIR_P&;t t $ac_datarootdir_hack " eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$ac_tmp/subs.awk" \ >$ac_tmp/out || as_fn_error $? "could not create $ac_file" "$LINENO" 5 test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ "$ac_tmp/out"`; test -z "$ac_out"; } && { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' which seems to be undefined. Please make sure it is defined" >&5 $as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' which seems to be undefined. Please make sure it is defined" >&2;} rm -f "$ac_tmp/stdin" case $ac_file in -) cat "$ac_tmp/out" && rm -f "$ac_tmp/out";; *) rm -f "$ac_file" && mv "$ac_tmp/out" "$ac_file";; esac \ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; :H) # # CONFIG_HEADER # if test x"$ac_file" != x-; then { $as_echo "/* $configure_input */" \ && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" } >"$ac_tmp/config.h" \ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 if diff "$ac_file" "$ac_tmp/config.h" >/dev/null 2>&1; then { $as_echo "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5 $as_echo "$as_me: $ac_file is unchanged" >&6;} else rm -f "$ac_file" mv "$ac_tmp/config.h" "$ac_file" \ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 fi else $as_echo "/* $configure_input */" \ && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" \ || as_fn_error $? "could not create -" "$LINENO" 5 fi # Compute "$ac_file"'s index in $config_headers. _am_arg="$ac_file" _am_stamp_count=1 for _am_header in $config_headers :; do case $_am_header in $_am_arg | $_am_arg:* ) break ;; * ) _am_stamp_count=`expr $_am_stamp_count + 1` ;; esac done echo "timestamp for $_am_arg" >`$as_dirname -- "$_am_arg" || $as_expr X"$_am_arg" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$_am_arg" : 'X\(//\)[^/]' \| \ X"$_am_arg" : 'X\(//\)$' \| \ X"$_am_arg" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$_am_arg" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'`/stamp-h$_am_stamp_count ;; :C) { $as_echo "$as_me:${as_lineno-$LINENO}: executing $ac_file commands" >&5 $as_echo "$as_me: executing $ac_file commands" >&6;} ;; esac case $ac_file$ac_mode in "depfiles":C) test x"$AMDEP_TRUE" != x"" || { # Older Autoconf quotes --file arguments for eval, but not when files # are listed without --file. Let's play safe and only enable the eval # if we detect the quoting. case $CONFIG_FILES in *\'*) eval set x "$CONFIG_FILES" ;; *) set x $CONFIG_FILES ;; esac shift for mf do # Strip MF so we end up with the name of the file. mf=`echo "$mf" | sed -e 's/:.*$//'` # Check whether this is an Automake generated Makefile or not. # We used to match only the files named 'Makefile.in', but # some people rename them; so instead we look at the file content. # Grep'ing the first line is not enough: some people post-process # each Makefile.in and add a new line on top of each file to say so. # Grep'ing the whole file is not good either: AIX grep has a line # limit of 2048, but all sed's we know have understand at least 4000. if sed -n 's,^#.*generated by automake.*,X,p' "$mf" | grep X >/dev/null 2>&1; then dirpart=`$as_dirname -- "$mf" || $as_expr X"$mf" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$mf" : 'X\(//\)[^/]' \| \ X"$mf" : 'X\(//\)$' \| \ X"$mf" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$mf" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` else continue fi # Extract the definition of DEPDIR, am__include, and am__quote # from the Makefile without running 'make'. DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` test -z "$DEPDIR" && continue am__include=`sed -n 's/^am__include = //p' < "$mf"` test -z "$am__include" && continue am__quote=`sed -n 's/^am__quote = //p' < "$mf"` # Find all dependency output files, they are included files with # $(DEPDIR) in their names. We invoke sed twice because it is the # simplest approach to changing $(DEPDIR) to its actual value in the # expansion. for file in `sed -n " s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g'`; do # Make sure the directory exists. test -f "$dirpart/$file" && continue fdir=`$as_dirname -- "$file" || $as_expr X"$file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$file" : 'X\(//\)[^/]' \| \ X"$file" : 'X\(//\)$' \| \ X"$file" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$file" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` as_dir=$dirpart/$fdir; as_fn_mkdir_p # echo "creating $dirpart/$file" echo '# dummy' > "$dirpart/$file" done done } ;; esac done # for ac_tag as_fn_exit 0 _ACEOF ac_clean_files=$ac_clean_files_save test $ac_write_fail = 0 || as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 # configure is writing to config.log, and then calls config.status. # config.status does its own redirection, appending to config.log. # Unfortunately, on DOS this fails, as config.log is still kept open # by configure, so config.status won't be able to write to it; its # output is simply discarded. So we exec the FD to /dev/null, # effectively closing config.log, so it can be properly (re)opened and # appended to by config.status. When coming back to configure, we # need to make the FD available again. if test "$no_create" != yes; then ac_cs_success=: ac_config_status_args= test "$silent" = yes && ac_config_status_args="$ac_config_status_args --quiet" exec 5>/dev/null $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false exec 5>>config.log # Use ||, not &&, to avoid exiting from the if with $? = 1, which # would make configure fail if this is the last instruction. $ac_cs_success || as_fn_exit 1 fi if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} fi memcached-1.5.22/assoc.h0000664000175000017500000000076713575022500011724 00000000000000/* associative array */ void assoc_init(const int hashpower_init); item *assoc_find(const char *key, const size_t nkey, const uint32_t hv); int assoc_insert(item *item, const uint32_t hv); void assoc_delete(const char *key, const size_t nkey, const uint32_t hv); void do_assoc_move_next_bucket(void); int start_assoc_maintenance_thread(void); void stop_assoc_maintenance_thread(void); void assoc_start_expand(uint64_t curr_items); extern unsigned int hashpower; extern unsigned int item_lock_hashpower; memcached-1.5.22/config.guess0000755000175000017500000012637313244306071012764 00000000000000#! /bin/sh # Attempt to guess a canonical system name. # Copyright 1992-2018 Free Software Foundation, Inc. timestamp='2018-02-24' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, see . # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that # program. This Exception is an additional permission under section 7 # of the GNU General Public License, version 3 ("GPLv3"). # # Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: # https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess # # Please send patches to . me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] Output the configuration name of the system \`$me' is run on. Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit Report bugs and patches to ." version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. Copyright 1992-2018 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" Try \`$me --help' for more information." # Parse command line while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) echo "$timestamp" ; exit ;; --version | -v ) echo "$version" ; exit ;; --help | --h* | -h ) echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. break ;; -* ) echo "$me: invalid option $1$help" >&2 exit 1 ;; * ) break ;; esac done if test $# != 0; then echo "$me: too many arguments$help" >&2 exit 1 fi trap 'exit 1' 1 2 15 # CC_FOR_BUILD -- compiler used by this script. Note that the use of a # compiler to aid in system detection is discouraged as it requires # temporary files to be created and, as you can see below, it is a # headache to deal with in a portable fashion. # Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still # use `HOST_CC' if defined, but it is deprecated. # Portable tmp directory creation inspired by the Autoconf team. set_cc_for_build=' trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; : ${TMPDIR=/tmp} ; { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; dummy=$tmp/dummy ; tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; case $CC_FOR_BUILD,$HOST_CC,$CC in ,,) echo "int x;" > "$dummy.c" ; for c in cc gcc c89 c99 ; do if ($c -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then CC_FOR_BUILD="$c"; break ; fi ; done ; if test x"$CC_FOR_BUILD" = x ; then CC_FOR_BUILD=no_compiler_found ; fi ;; ,,*) CC_FOR_BUILD=$CC ;; ,*,*) CC_FOR_BUILD=$HOST_CC ;; esac ; set_cc_for_build= ;' # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) if (test -f /.attbin/uname) >/dev/null 2>&1 ; then PATH=$PATH:/.attbin ; export PATH fi UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown case "$UNAME_SYSTEM" in Linux|GNU|GNU/*) # If the system lacks a compiler, then just pick glibc. # We could probably try harder. LIBC=gnu eval "$set_cc_for_build" cat <<-EOF > "$dummy.c" #include #if defined(__UCLIBC__) LIBC=uclibc #elif defined(__dietlibc__) LIBC=dietlibc #else LIBC=gnu #endif EOF eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" # If ldd exists, use it to detect musl libc. if command -v ldd >/dev/null && \ ldd --version 2>&1 | grep -q ^musl then LIBC=musl fi ;; esac # Note: order is significant - the case branches are not exclusive. case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently # switched to ELF, *-*-netbsd* would select the old # object file format. This provides both forward # compatibility and a consistent mechanism for selecting the # object file format. # # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ "/sbin/$sysctl" 2>/dev/null || \ "/usr/sbin/$sysctl" 2>/dev/null || \ echo unknown)` case "$UNAME_MACHINE_ARCH" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; earmv*) arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` machine="${arch}${endian}"-unknown ;; *) machine="$UNAME_MACHINE_ARCH"-unknown ;; esac # The Operating System including object format, if it has switched # to ELF recently (or will in the future) and ABI. case "$UNAME_MACHINE_ARCH" in earm*) os=netbsdelf ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval "$set_cc_for_build" if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ then # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). # Return netbsd for either. FIX? os=netbsd else os=netbsdelf fi ;; *) os=netbsd ;; esac # Determine ABI tags. case "$UNAME_MACHINE_ARCH" in earm*) expr='s/^earmv[0-9]/-eabi/;s/eb$//' abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` ;; esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need # kernel version information, so it can be replaced with a # suitable tag, in the style of linux-gnu. case "$UNAME_VERSION" in Debian*) release='-gnu' ;; *) release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. echo "$machine-${os}${release}${abi}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" exit ;; *:OpenBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" exit ;; *:LibertyBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" exit ;; *:MidnightBSD:*:*) echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" exit ;; *:ekkoBSD:*:*) echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" exit ;; *:SolidBSD:*:*) echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" exit ;; macppc:MirBSD:*:*) echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; *:MirBSD:*:*) echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" exit ;; *:Sortix:*:*) echo "$UNAME_MACHINE"-unknown-sortix exit ;; *:Redox:*:*) echo "$UNAME_MACHINE"-unknown-redox exit ;; mips:OSF1:*.*) echo mips-dec-osf1 exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` ;; *5.*) UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on # OSF/1 and Tru64 systems produced since 1995. I hope that # covers most systems running today. This code pipes the CPU # types through head -n 1, so we only detect the type of CPU 0. ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") UNAME_MACHINE=alpha ;; "EV4.5 (21064)") UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") UNAME_MACHINE=alpha ;; "EV5 (21164)") UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 exit $exitcode ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 exit ;; *:[Aa]miga[Oo][Ss]:*:*) echo "$UNAME_MACHINE"-unknown-amigaos exit ;; *:[Mm]orph[Oo][Ss]:*:*) echo "$UNAME_MACHINE"-unknown-morphos exit ;; *:OS/390:*:*) echo i370-ibm-openedition exit ;; *:z/VM:*:*) echo s390-ibm-zvmoe exit ;; *:OS400:*:*) echo powerpc-ibm-os400 exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) echo arm-acorn-riscix"$UNAME_RELEASE" exit ;; arm*:riscos:*:*|arm*:RISCOS:*:*) echo arm-unknown-riscos exit ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) echo hppa1.1-hitachi-hiuxmpp exit ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. if test "`(/bin/universe) 2>/dev/null`" = att ; then echo pyramid-pyramid-sysv3 else echo pyramid-pyramid-bsd fi exit ;; NILE*:*:*:dcosx) echo pyramid-pyramid-svr4 exit ;; DRS?6000:unix:4.0:6*) echo sparc-icl-nx6 exit ;; DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) case `/usr/bin/uname -p` in sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; sun4H:SunOS:5.*:*) echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) eval "$set_cc_for_build" SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then SUN_ARCH=x86_64 fi fi echo "$SUN_ARCH"-pc-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in Series*|S4*) UNAME_RELEASE=`uname -v` ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" exit ;; sun3*:SunOS:*:*) echo m68k-sun-sunos"$UNAME_RELEASE" exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) echo m68k-sun-sunos"$UNAME_RELEASE" ;; sun4) echo sparc-sun-sunos"$UNAME_RELEASE" ;; esac exit ;; aushp:SunOS:*:*) echo sparc-auspex-sunos"$UNAME_RELEASE" exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not # "atarist" or "atariste" at least should have a processor # > m68000). The system name ranges from "MiNT" over "FreeMiNT" # to the lowercase version "mint" (or "freemint"). Finally # the system name "TOS" denotes a system which is actually not # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) echo m68k-atari-mint"$UNAME_RELEASE" exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) echo m68k-atari-mint"$UNAME_RELEASE" exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) echo m68k-atari-mint"$UNAME_RELEASE" exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) echo m68k-milan-mint"$UNAME_RELEASE" exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) echo m68k-hades-mint"$UNAME_RELEASE" exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) echo m68k-unknown-mint"$UNAME_RELEASE" exit ;; m68k:machten:*:*) echo m68k-apple-machten"$UNAME_RELEASE" exit ;; powerpc:machten:*:*) echo powerpc-apple-machten"$UNAME_RELEASE" exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 exit ;; RISC*:ULTRIX:*:*) echo mips-dec-ultrix"$UNAME_RELEASE" exit ;; VAX*:ULTRIX*:*:*) echo vax-dec-ultrix"$UNAME_RELEASE" exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) echo clipper-intergraph-clix"$UNAME_RELEASE" exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) eval "$set_cc_for_build" sed 's/^ //' << EOF > "$dummy.c" #ifdef __cplusplus #include /* for printf() prototype */ int main (int argc, char *argv[]) { #else int main (argc, argv) int argc; char *argv[]; { #endif #if defined (host_mips) && defined (MIPSEB) #if defined (SYSTYPE_SYSV) printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_SVR4) printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); #endif #endif exit (-1); } EOF $CC_FOR_BUILD -o "$dummy" "$dummy.c" && dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && SYSTEM_NAME=`"$dummy" "$dummyarg"` && { echo "$SYSTEM_NAME"; exit; } echo mips-mips-riscos"$UNAME_RELEASE" exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax exit ;; Motorola:*:4.3:PL8-*) echo powerpc-harris-powermax exit ;; Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) echo powerpc-harris-powermax exit ;; Night_Hawk:Power_UNIX:*:*) echo powerpc-harris-powerunix exit ;; m88k:CX/UX:7*:*) echo m88k-harris-cxux7 exit ;; m88k:*:4*:R4*) echo m88k-motorola-sysv4 exit ;; m88k:*:3*:R3*) echo m88k-motorola-sysv3 exit ;; AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures UNAME_PROCESSOR=`/usr/bin/uname -p` if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] then if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ [ "$TARGET_BINARY_INTERFACE"x = x ] then echo m88k-dg-dgux"$UNAME_RELEASE" else echo m88k-dg-dguxbcs"$UNAME_RELEASE" fi else echo i586-dg-dgux"$UNAME_RELEASE" fi exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) echo m88k-dolphin-sysv3 exit ;; M88*:*:R3*:*) # Delta 88k system running SVR3 echo m88k-motorola-sysv3 exit ;; XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) echo m88k-tektronix-sysv3 exit ;; Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' i*86:AIX:*:*) echo i386-ibm-aix exit ;; ia64:AIX:*:*) if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` else IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then eval "$set_cc_for_build" sed 's/^ //' << EOF > "$dummy.c" #include main() { if (!__power_pc()) exit(1); puts("powerpc-ibm-aix3.2.5"); exit(0); } EOF if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` then echo "$SYSTEM_NAME" else echo rs6000-ibm-aix3.2.5 fi elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then echo rs6000-ibm-aix3.2.4 else echo rs6000-ibm-aix3.2 fi exit ;; *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc fi if [ -x /usr/bin/lslpp ] ; then IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi echo "$IBM_ARCH"-ibm-aix"$IBM_REV" exit ;; *:AIX:*:*) echo rs6000-ibm-aix exit ;; ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) echo romp-ibm-bsd4.4 exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx exit ;; DPX/2?00:B.O.S.:*:*) echo m68k-bull-sysv3 exit ;; 9000/[34]??:4.3bsd:1.*:*) echo m68k-hp-bsd exit ;; hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` case "$UNAME_MACHINE" in 9000/31?) HP_ARCH=m68000 ;; 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` case "$sc_cpu_version" in 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 case "$sc_kernel_bits" in 32) HP_ARCH=hppa2.0n ;; 64) HP_ARCH=hppa2.0w ;; '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi if [ "$HP_ARCH" = "" ]; then eval "$set_cc_for_build" sed 's/^ //' << EOF > "$dummy.c" #define _HPUX_SOURCE #include #include int main () { #if defined(_SC_KERNEL_BITS) long bits = sysconf(_SC_KERNEL_BITS); #endif long cpu = sysconf (_SC_CPU_VERSION); switch (cpu) { case CPU_PA_RISC1_0: puts ("hppa1.0"); break; case CPU_PA_RISC1_1: puts ("hppa1.1"); break; case CPU_PA_RISC2_0: #if defined(_SC_KERNEL_BITS) switch (bits) { case 64: puts ("hppa2.0w"); break; case 32: puts ("hppa2.0n"); break; default: puts ("hppa2.0"); break; } break; #else /* !defined(_SC_KERNEL_BITS) */ puts ("hppa2.0"); break; #endif default: puts ("hppa1.0"); break; } exit (0); } EOF (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac if [ "$HP_ARCH" = hppa2.0w ] then eval "$set_cc_for_build" # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler # generating 64-bit code. GNU and HP use different nomenclature: # # $ CC_FOR_BUILD=cc ./config.guess # => hppa2.0w-hp-hpux11.23 # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then HP_ARCH=hppa2.0w else HP_ARCH=hppa64 fi fi echo "$HP_ARCH"-hp-hpux"$HPUX_REV" exit ;; ia64:HP-UX:*:*) HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) eval "$set_cc_for_build" sed 's/^ //' << EOF > "$dummy.c" #include int main () { long cpu = sysconf (_SC_CPU_VERSION); /* The order matters, because CPU_IS_HP_MC68K erroneously returns true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct results, however. */ if (CPU_IS_PA_RISC (cpu)) { switch (cpu) { case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; default: puts ("hppa-hitachi-hiuxwe2"); break; } } else if (CPU_IS_HP_MC68K (cpu)) puts ("m68k-hitachi-hiuxwe2"); else puts ("unknown-hitachi-hiuxwe2"); exit (0); } EOF $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) echo hppa1.1-hp-bsd exit ;; 9000/8??:4.3bsd:*:*) echo hppa1.0-hp-bsd exit ;; *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix exit ;; hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) echo hppa1.1-hp-osf exit ;; hp8??:OSF1:*:*) echo hppa1.0-hp-osf exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then echo "$UNAME_MACHINE"-unknown-osf1mk else echo "$UNAME_MACHINE"-unknown-osf1 fi exit ;; parisc*:Lites*:*:*) echo hppa1.1-hp-lites exit ;; C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) echo c1-convex-bsd exit ;; C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi exit ;; C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) echo c34-convex-bsd exit ;; C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) echo c38-convex-bsd exit ;; C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) echo c4-convex-bsd exit ;; CRAY*Y-MP:*:*:*) echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*[A-Z]90:*:*:*) echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' exit ;; CRAY*TS:*:*:*) echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*T3E:*:*:*) echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*SV1:*:*:*) echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; *:UNICOS/mp:*:*) echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" exit ;; sparc*:BSD/OS:*:*) echo sparc-unknown-bsdi"$UNAME_RELEASE" exit ;; *:BSD/OS:*:*) echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" exit ;; *:FreeBSD:*:*) UNAME_PROCESSOR=`/usr/bin/uname -p` case "$UNAME_PROCESSOR" in amd64) UNAME_PROCESSOR=x86_64 ;; i386) UNAME_PROCESSOR=i586 ;; esac echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; i*:CYGWIN*:*) echo "$UNAME_MACHINE"-pc-cygwin exit ;; *:MINGW64*:*) echo "$UNAME_MACHINE"-pc-mingw64 exit ;; *:MINGW*:*) echo "$UNAME_MACHINE"-pc-mingw32 exit ;; *:MSYS*:*) echo "$UNAME_MACHINE"-pc-msys exit ;; i*:PW*:*) echo "$UNAME_MACHINE"-pc-pw32 exit ;; *:Interix*:*) case "$UNAME_MACHINE" in x86) echo i586-pc-interix"$UNAME_RELEASE" exit ;; authenticamd | genuineintel | EM64T) echo x86_64-unknown-interix"$UNAME_RELEASE" exit ;; IA64) echo ia64-unknown-interix"$UNAME_RELEASE" exit ;; esac ;; i*:UWIN*:*) echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) echo x86_64-unknown-cygwin exit ;; prep*:SunOS:5.*:*) echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; *:GNU:*:*) # the GNU system echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" exit ;; i*86:Minix:*:*) echo "$UNAME_MACHINE"-pc-minix exit ;; aarch64:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in EV5) UNAME_MACHINE=alphaev5 ;; EV56) UNAME_MACHINE=alphaev56 ;; PCA56) UNAME_MACHINE=alphapca56 ;; PCA57) UNAME_MACHINE=alphapca56 ;; EV6) UNAME_MACHINE=alphaev6 ;; EV67) UNAME_MACHINE=alphaev67 ;; EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 if test "$?" = 0 ; then LIBC=gnulibc1 ; fi echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arc:Linux:*:* | arceb:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arm*:Linux:*:*) eval "$set_cc_for_build" if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi else echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf fi fi exit ;; avr32*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; cris:Linux:*:*) echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; crisv32:Linux:*:*) echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; e2k:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; frv:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; hexagon:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:Linux:*:*) echo "$UNAME_MACHINE"-pc-linux-"$LIBC" exit ;; ia64:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; k1om:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m32r*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m68*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; mips:Linux:*:* | mips64:Linux:*:*) eval "$set_cc_for_build" sed 's/^ //' << EOF > "$dummy.c" #undef CPU #undef ${UNAME_MACHINE} #undef ${UNAME_MACHINE}el #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) CPU=${UNAME_MACHINE}el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) CPU=${UNAME_MACHINE} #else CPU= #endif #endif EOF eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`" test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; } ;; mips64el:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; openrisc*:Linux:*:*) echo or1k-unknown-linux-"$LIBC" exit ;; or32:Linux:*:* | or1k*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; padre:Linux:*:*) echo sparc-unknown-linux-"$LIBC" exit ;; parisc64:Linux:*:* | hppa64:Linux:*:*) echo hppa64-unknown-linux-"$LIBC" exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; *) echo hppa-unknown-linux-"$LIBC" ;; esac exit ;; ppc64:Linux:*:*) echo powerpc64-unknown-linux-"$LIBC" exit ;; ppc:Linux:*:*) echo powerpc-unknown-linux-"$LIBC" exit ;; ppc64le:Linux:*:*) echo powerpc64le-unknown-linux-"$LIBC" exit ;; ppcle:Linux:*:*) echo powerpcle-unknown-linux-"$LIBC" exit ;; riscv32:Linux:*:* | riscv64:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" exit ;; sh64*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sh*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; tile*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; vax:Linux:*:*) echo "$UNAME_MACHINE"-dec-linux-"$LIBC" exit ;; x86_64:Linux:*:*) if objdump -f /bin/sh | grep -q elf32-x86-64; then echo "$UNAME_MACHINE"-pc-linux-"$LIBC"x32 else echo "$UNAME_MACHINE"-pc-linux-"$LIBC" fi exit ;; xtensa*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. # earlier versions are messed up and put the nodename in both # sysname and nodename. echo i386-sequent-sysv4 exit ;; i*86:UNIX_SV:4.2MP:2.*) # Unixware is an offshoot of SVR4, but it has its own version # number series starting with 2... # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. # Use sysv4.2uw... so that sysv4* matches it. echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. echo "$UNAME_MACHINE"-pc-os2-emx exit ;; i*86:XTS-300:*:STOP) echo "$UNAME_MACHINE"-unknown-stop exit ;; i*86:atheos:*:*) echo "$UNAME_MACHINE"-unknown-atheos exit ;; i*86:syllable:*:*) echo "$UNAME_MACHINE"-pc-syllable exit ;; i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) echo i386-unknown-lynxos"$UNAME_RELEASE" exit ;; i*86:*DOS:*:*) echo "$UNAME_MACHINE"-pc-msdosdjgpp exit ;; i*86:*:4.*:*) UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" else echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" fi exit ;; i*86:*:5:[678]*) # UnixWare 7.x, OpenUNIX and OpenServer 6. case `/bin/uname -X | grep "^Machine"` in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ && UNAME_MACHINE=i586 (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ && UNAME_MACHINE=i686 (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ && UNAME_MACHINE=i686 echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" else echo "$UNAME_MACHINE"-pc-sysv32 fi exit ;; pc:*:*:*) # Left here for compatibility: # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; Intel:Mach:3*:*) echo i386-pc-mach3 exit ;; paragon:*:*:*) echo i860-intel-osf1 exit ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 fi exit ;; mini*:CTIX:SYS*5:*) # "miniframe" echo m68010-convergent-sysv exit ;; mc68k:UNIX:SYSTEM5:3.51m) echo m68k-convergent-sysv exit ;; M680?0:D-NIX:5.3:*) echo m68k-diab-dnix exit ;; M68*:*:R3V[5678]*:*) test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) OS_REL='' test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4; exit; } ;; NCR*:*:4.2:* | MPRAS*:*:4.2:*) OS_REL='.3' test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) echo m68k-unknown-lynxos"$UNAME_RELEASE" exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 exit ;; TSUNAMI:LynxOS:2.*:*) echo sparc-unknown-lynxos"$UNAME_RELEASE" exit ;; rs6000:LynxOS:2.*:*) echo rs6000-unknown-lynxos"$UNAME_RELEASE" exit ;; PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) echo powerpc-unknown-lynxos"$UNAME_RELEASE" exit ;; SM[BE]S:UNIX_SV:*:*) echo mips-dde-sysv"$UNAME_RELEASE" exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 exit ;; RM*:SINIX-*:*:*) echo mips-sni-sysv4 exit ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` echo "$UNAME_MACHINE"-sni-sysv4 else echo ns32k-sni-sysv fi exit ;; PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort # says echo i586-unisys-sysv4 exit ;; *:UNIX_System_V:4*:FTX*) # From Gerald Hewes . # How about differentiating between stratus architectures? -djm echo hppa1.1-stratus-sysv4 exit ;; *:*:*:FTX*) # From seanf@swdc.stratus.com. echo i860-stratus-sysv4 exit ;; i*86:VOS:*:*) # From Paul.Green@stratus.com. echo "$UNAME_MACHINE"-stratus-vos exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos exit ;; mc68*:A/UX:*:*) echo m68k-apple-aux"$UNAME_RELEASE" exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then echo mips-nec-sysv"$UNAME_RELEASE" else echo mips-unknown-sysv"$UNAME_RELEASE" fi exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. echo powerpc-be-beos exit ;; BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. echo powerpc-apple-beos exit ;; BePC:BeOS:*:*) # BeOS running on Intel PC compatible. echo i586-pc-beos exit ;; BePC:Haiku:*:*) # Haiku running on Intel PC compatible. echo i586-pc-haiku exit ;; x86_64:Haiku:*:*) echo x86_64-unknown-haiku exit ;; SX-4:SUPER-UX:*:*) echo sx4-nec-superux"$UNAME_RELEASE" exit ;; SX-5:SUPER-UX:*:*) echo sx5-nec-superux"$UNAME_RELEASE" exit ;; SX-6:SUPER-UX:*:*) echo sx6-nec-superux"$UNAME_RELEASE" exit ;; SX-7:SUPER-UX:*:*) echo sx7-nec-superux"$UNAME_RELEASE" exit ;; SX-8:SUPER-UX:*:*) echo sx8-nec-superux"$UNAME_RELEASE" exit ;; SX-8R:SUPER-UX:*:*) echo sx8r-nec-superux"$UNAME_RELEASE" exit ;; SX-ACE:SUPER-UX:*:*) echo sxace-nec-superux"$UNAME_RELEASE" exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Rhapsody:*:*) echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Darwin:*:*) UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown eval "$set_cc_for_build" if test "$UNAME_PROCESSOR" = unknown ; then UNAME_PROCESSOR=powerpc fi if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then case $UNAME_PROCESSOR in i386) UNAME_PROCESSOR=x86_64 ;; powerpc) UNAME_PROCESSOR=powerpc64 ;; esac fi # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_PPC >/dev/null then UNAME_PROCESSOR=powerpc fi fi elif test "$UNAME_PROCESSOR" = i386 ; then # Avoid executing cc on OS X 10.9, as it ships with a stub # that puts up a graphical alert prompting to install # developer tools. Any system running Mac OS X 10.7 or # later (Darwin 11 and later) is required to have a 64-bit # processor. This is not true of the ARM version of Darwin # that Apple uses in portable devices. UNAME_PROCESSOR=x86_64 fi echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" exit ;; *:QNX:*:4*) echo i386-pc-qnx exit ;; NEO-*:NONSTOP_KERNEL:*:*) echo neo-tandem-nsk"$UNAME_RELEASE" exit ;; NSE-*:NONSTOP_KERNEL:*:*) echo nse-tandem-nsk"$UNAME_RELEASE" exit ;; NSR-*:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk"$UNAME_RELEASE" exit ;; NSV-*:NONSTOP_KERNEL:*:*) echo nsv-tandem-nsk"$UNAME_RELEASE" exit ;; NSX-*:NONSTOP_KERNEL:*:*) echo nsx-tandem-nsk"$UNAME_RELEASE" exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux exit ;; BS2000:POSIX*:*:*) echo bs2000-siemens-sysv exit ;; DS/*:UNIX_System_V:*:*) echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" fi echo "$UNAME_MACHINE"-unknown-plan9 exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 exit ;; *:TENEX:*:*) echo pdp10-unknown-tenex exit ;; KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) echo pdp10-dec-tops20 exit ;; XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) echo pdp10-xkl-tops20 exit ;; *:TOPS-20:*:*) echo pdp10-unknown-tops20 exit ;; *:ITS:*:*) echo pdp10-unknown-its exit ;; SEI:*:*:SEIUX) echo mips-sei-seiux"$UNAME_RELEASE" exit ;; *:DragonFly:*:*) echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; *:*VMS:*:*) UNAME_MACHINE=`(uname -p) 2>/dev/null` case "$UNAME_MACHINE" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; V*) echo vax-dec-vms ; exit ;; esac ;; *:XENIX:*:SysV) echo i386-pc-xenix exit ;; i*86:skyos:*:*) echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" exit ;; i*86:rdos:*:*) echo "$UNAME_MACHINE"-pc-rdos exit ;; i*86:AROS:*:*) echo "$UNAME_MACHINE"-pc-aros exit ;; x86_64:VMkernel:*:*) echo "$UNAME_MACHINE"-unknown-esx exit ;; amd64:Isilon\ OneFS:*:*) echo x86_64-unknown-onefs exit ;; esac echo "$0: unable to guess system type" >&2 case "$UNAME_MACHINE:$UNAME_SYSTEM" in mips:Linux | mips64:Linux) # If we got here on MIPS GNU/Linux, output extra information. cat >&2 <&2 </dev/null || echo unknown` uname -r = `(uname -r) 2>/dev/null || echo unknown` uname -s = `(uname -s) 2>/dev/null || echo unknown` uname -v = `(uname -v) 2>/dev/null || echo unknown` /usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` /bin/uname -X = `(/bin/uname -X) 2>/dev/null` hostinfo = `(hostinfo) 2>/dev/null` /bin/universe = `(/bin/universe) 2>/dev/null` /usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` /bin/arch = `(/bin/arch) 2>/dev/null` /usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` UNAME_MACHINE = "$UNAME_MACHINE" UNAME_RELEASE = "$UNAME_RELEASE" UNAME_SYSTEM = "$UNAME_SYSTEM" UNAME_VERSION = "$UNAME_VERSION" EOF exit 1 # Local variables: # eval: (add-hook 'write-file-functions 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" # End: memcached-1.5.22/trace.h0000664000175000017500000000613713461170555011717 00000000000000#ifndef TRACE_H #define TRACE_H #ifdef ENABLE_DTRACE #include "memcached_dtrace.h" #else #define MEMCACHED_ASSOC_DELETE(arg0, arg1) #define MEMCACHED_ASSOC_DELETE_ENABLED() (0) #define MEMCACHED_ASSOC_FIND(arg0, arg1, arg2) #define MEMCACHED_ASSOC_FIND_ENABLED() (0) #define MEMCACHED_ASSOC_INSERT(arg0, arg1) #define MEMCACHED_ASSOC_INSERT_ENABLED() (0) #define MEMCACHED_COMMAND_ADD(arg0, arg1, arg2, arg3, arg4) #define MEMCACHED_COMMAND_ADD_ENABLED() (0) #define MEMCACHED_COMMAND_APPEND(arg0, arg1, arg2, arg3, arg4) #define MEMCACHED_COMMAND_APPEND_ENABLED() (0) #define MEMCACHED_COMMAND_CAS(arg0, arg1, arg2, arg3, arg4) #define MEMCACHED_COMMAND_CAS_ENABLED() (0) #define MEMCACHED_COMMAND_DECR(arg0, arg1, arg2, arg3) #define MEMCACHED_COMMAND_DECR_ENABLED() (0) #define MEMCACHED_COMMAND_DELETE(arg0, arg1, arg2) #define MEMCACHED_COMMAND_DELETE_ENABLED() (0) #define MEMCACHED_COMMAND_GET(arg0, arg1, arg2, arg3, arg4) #define MEMCACHED_COMMAND_GET_ENABLED() (0) #define MEMCACHED_COMMAND_TOUCH(arg0, arg1, arg2, arg3, arg4) #define MEMCACHED_COMMAND_TOUCH_ENABLED() (0) #define MEMCACHED_COMMAND_INCR(arg0, arg1, arg2, arg3) #define MEMCACHED_COMMAND_INCR_ENABLED() (0) #define MEMCACHED_COMMAND_PREPEND(arg0, arg1, arg2, arg3, arg4) #define MEMCACHED_COMMAND_PREPEND_ENABLED() (0) #define MEMCACHED_COMMAND_REPLACE(arg0, arg1, arg2, arg3, arg4) #define MEMCACHED_COMMAND_REPLACE_ENABLED() (0) #define MEMCACHED_COMMAND_SET(arg0, arg1, arg2, arg3, arg4) #define MEMCACHED_COMMAND_SET_ENABLED() (0) #define MEMCACHED_CONN_ALLOCATE(arg0) #define MEMCACHED_CONN_ALLOCATE_ENABLED() (0) #define MEMCACHED_CONN_CREATE(arg0) #define MEMCACHED_CONN_CREATE_ENABLED() (0) #define MEMCACHED_CONN_DESTROY(arg0) #define MEMCACHED_CONN_DESTROY_ENABLED() (0) #define MEMCACHED_CONN_DISPATCH(arg0, arg1) #define MEMCACHED_CONN_DISPATCH_ENABLED() (0) #define MEMCACHED_CONN_RELEASE(arg0) #define MEMCACHED_CONN_RELEASE_ENABLED() (0) #define MEMCACHED_ITEM_LINK(arg0, arg1, arg2) #define MEMCACHED_ITEM_LINK_ENABLED() (0) #define MEMCACHED_ITEM_REMOVE(arg0, arg1, arg2) #define MEMCACHED_ITEM_REMOVE_ENABLED() (0) #define MEMCACHED_ITEM_REPLACE(arg0, arg1, arg2, arg3, arg4, arg5) #define MEMCACHED_ITEM_REPLACE_ENABLED() (0) #define MEMCACHED_ITEM_UNLINK(arg0, arg1, arg2) #define MEMCACHED_ITEM_UNLINK_ENABLED() (0) #define MEMCACHED_ITEM_UPDATE(arg0, arg1, arg2) #define MEMCACHED_ITEM_UPDATE_ENABLED() (0) #define MEMCACHED_PROCESS_COMMAND_END(arg0, arg1, arg2) #define MEMCACHED_PROCESS_COMMAND_END_ENABLED() (0) #define MEMCACHED_PROCESS_COMMAND_START(arg0, arg1, arg2) #define MEMCACHED_PROCESS_COMMAND_START_ENABLED() (0) #define MEMCACHED_SLABS_ALLOCATE(arg0, arg1, arg2, arg3) #define MEMCACHED_SLABS_ALLOCATE_ENABLED() (0) #define MEMCACHED_SLABS_ALLOCATE_FAILED(arg0, arg1) #define MEMCACHED_SLABS_ALLOCATE_FAILED_ENABLED() (0) #define MEMCACHED_SLABS_FREE(arg0, arg1, arg2) #define MEMCACHED_SLABS_FREE_ENABLED() (0) #define MEMCACHED_SLABS_SLABCLASS_ALLOCATE(arg0) #define MEMCACHED_SLABS_SLABCLASS_ALLOCATE_ENABLED() (0) #define MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED(arg0) #define MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED_ENABLED() (0) #endif #endif memcached-1.5.22/version.m40000664000175000017500000000004613615454042012365 00000000000000m4_define([VERSION_NUMBER], [1.5.22]) memcached-1.5.22/extstore.c0000664000175000017500000007506013611703161012462 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ // FIXME: config.h? #include #include // end FIXME #include #include #include #include #include #include #include #include #include #include #include #include "extstore.h" // TODO: better if an init option turns this on/off. #ifdef EXTSTORE_DEBUG #define E_DEBUG(...) \ do { \ fprintf(stderr, __VA_ARGS__); \ } while (0) #else #define E_DEBUG(...) #endif #define STAT_L(e) pthread_mutex_lock(&e->stats_mutex); #define STAT_UL(e) pthread_mutex_unlock(&e->stats_mutex); #define STAT_INCR(e, stat, amount) { \ pthread_mutex_lock(&e->stats_mutex); \ e->stats.stat += amount; \ pthread_mutex_unlock(&e->stats_mutex); \ } #define STAT_DECR(e, stat, amount) { \ pthread_mutex_lock(&e->stats_mutex); \ e->stats.stat -= amount; \ pthread_mutex_unlock(&e->stats_mutex); \ } typedef struct __store_wbuf { struct __store_wbuf *next; char *buf; char *buf_pos; unsigned int free; unsigned int size; unsigned int offset; /* offset into page this write starts at */ bool full; /* done writing to this page */ bool flushed; /* whether wbuf has been flushed to disk */ } _store_wbuf; typedef struct _store_page { pthread_mutex_t mutex; /* Need to be held for most operations */ uint64_t obj_count; /* _delete can decrease post-closing */ uint64_t bytes_used; /* _delete can decrease post-closing */ uint64_t offset; /* starting address of page within fd */ unsigned int version; unsigned int refcount; unsigned int allocated; unsigned int written; /* item offsets can be past written if wbuf not flushed */ unsigned int bucket; /* which bucket the page is linked into */ unsigned int free_bucket; /* which bucket this page returns to when freed */ int fd; unsigned short id; bool active; /* actively being written to */ bool closed; /* closed and draining before free */ bool free; /* on freelist */ _store_wbuf *wbuf; /* currently active wbuf from the stack */ struct _store_page *next; } store_page; typedef struct store_engine store_engine; typedef struct { pthread_mutex_t mutex; pthread_cond_t cond; obj_io *queue; store_engine *e; unsigned int depth; // queue depth } store_io_thread; typedef struct { pthread_mutex_t mutex; pthread_cond_t cond; store_engine *e; } store_maint_thread; struct store_engine { pthread_mutex_t mutex; /* covers internal stacks and variables */ store_page *pages; /* directly addressable page list */ _store_wbuf *wbuf_stack; /* wbuf freelist */ obj_io *io_stack; /* IO's to use with submitting wbuf's */ store_io_thread *io_threads; store_maint_thread *maint_thread; store_page *page_freelist; store_page **page_buckets; /* stack of pages currently allocated to each bucket */ store_page **free_page_buckets; /* stack of use-case isolated free pages */ size_t page_size; unsigned int version; /* global version counter */ unsigned int last_io_thread; /* round robin the IO threads */ unsigned int io_threadcount; /* count of IO threads */ unsigned int page_count; unsigned int page_free; /* unallocated pages */ unsigned int page_bucketcount; /* count of potential page buckets */ unsigned int free_page_bucketcount; /* count of free page buckets */ unsigned int io_depth; /* FIXME: Might cache into thr struct */ pthread_mutex_t stats_mutex; struct extstore_stats stats; }; static _store_wbuf *wbuf_new(size_t size) { _store_wbuf *b = calloc(1, sizeof(_store_wbuf)); if (b == NULL) return NULL; b->buf = malloc(size); if (b->buf == NULL) { free(b); return NULL; } b->buf_pos = b->buf; b->free = size; b->size = size; return b; } static store_io_thread *_get_io_thread(store_engine *e) { int tid = -1; long long int low = LLONG_MAX; pthread_mutex_lock(&e->mutex); // find smallest queue. ignoring lock since being wrong isn't fatal. // TODO: if average queue depth can be quickly tracked, can break as soon // as we see a thread that's less than average, and start from last_io_thread for (int x = 0; x < e->io_threadcount; x++) { if (e->io_threads[x].depth == 0) { tid = x; break; } else if (e->io_threads[x].depth < low) { tid = x; low = e->io_threads[x].depth; } } pthread_mutex_unlock(&e->mutex); return &e->io_threads[tid]; } static uint64_t _next_version(store_engine *e) { return e->version++; } static void *extstore_io_thread(void *arg); static void *extstore_maint_thread(void *arg); /* Copies stats internal to engine and computes any derived values */ void extstore_get_stats(void *ptr, struct extstore_stats *st) { store_engine *e = (store_engine *)ptr; STAT_L(e); memcpy(st, &e->stats, sizeof(struct extstore_stats)); STAT_UL(e); // grab pages_free/pages_used pthread_mutex_lock(&e->mutex); st->pages_free = e->page_free; st->pages_used = e->page_count - e->page_free; pthread_mutex_unlock(&e->mutex); st->io_queue = 0; for (int x = 0; x < e->io_threadcount; x++) { pthread_mutex_lock(&e->io_threads[x].mutex); st->io_queue += e->io_threads[x].depth; pthread_mutex_unlock(&e->io_threads[x].mutex); } // calculate bytes_fragmented. // note that open and yet-filled pages count against fragmentation. st->bytes_fragmented = st->pages_used * e->page_size - st->bytes_used; } void extstore_get_page_data(void *ptr, struct extstore_stats *st) { store_engine *e = (store_engine *)ptr; STAT_L(e); memcpy(st->page_data, e->stats.page_data, sizeof(struct extstore_page_data) * e->page_count); STAT_UL(e); } const char *extstore_err(enum extstore_res res) { const char *rv = "unknown error"; switch (res) { case EXTSTORE_INIT_BAD_WBUF_SIZE: rv = "page_size must be divisible by wbuf_size"; break; case EXTSTORE_INIT_NEED_MORE_WBUF: rv = "wbuf_count must be >= page_buckets"; break; case EXTSTORE_INIT_NEED_MORE_BUCKETS: rv = "page_buckets must be > 0"; break; case EXTSTORE_INIT_PAGE_WBUF_ALIGNMENT: rv = "page_size and wbuf_size must be divisible by 1024*1024*2"; break; case EXTSTORE_INIT_TOO_MANY_PAGES: rv = "page_count must total to < 65536. Increase page_size or lower path sizes"; break; case EXTSTORE_INIT_OOM: rv = "failed calloc for engine"; break; case EXTSTORE_INIT_OPEN_FAIL: rv = "failed to open file"; break; case EXTSTORE_INIT_THREAD_FAIL: break; } return rv; } // TODO: #define's for DEFAULT_BUCKET, FREE_VERSION, etc void *extstore_init(struct extstore_conf_file *fh, struct extstore_conf *cf, enum extstore_res *res) { int i; struct extstore_conf_file *f = NULL; pthread_t thread; if (cf->page_size % cf->wbuf_size != 0) { *res = EXTSTORE_INIT_BAD_WBUF_SIZE; return NULL; } // Should ensure at least one write buffer per potential page if (cf->page_buckets > cf->wbuf_count) { *res = EXTSTORE_INIT_NEED_MORE_WBUF; return NULL; } if (cf->page_buckets < 1) { *res = EXTSTORE_INIT_NEED_MORE_BUCKETS; return NULL; } // TODO: More intelligence around alignment of flash erasure block sizes if (cf->page_size % (1024 * 1024 * 2) != 0 || cf->wbuf_size % (1024 * 1024 * 2) != 0) { *res = EXTSTORE_INIT_PAGE_WBUF_ALIGNMENT; return NULL; } store_engine *e = calloc(1, sizeof(store_engine)); if (e == NULL) { *res = EXTSTORE_INIT_OOM; return NULL; } e->page_size = cf->page_size; uint64_t temp_page_count = 0; for (f = fh; f != NULL; f = f->next) { f->fd = open(f->file, O_RDWR | O_CREAT | O_TRUNC, 0644); if (f->fd < 0) { *res = EXTSTORE_INIT_OPEN_FAIL; #ifdef EXTSTORE_DEBUG perror("open"); #endif free(e); return NULL; } temp_page_count += f->page_count; f->offset = 0; } if (temp_page_count >= UINT16_MAX) { *res = EXTSTORE_INIT_TOO_MANY_PAGES; free(e); return NULL; } e->page_count = temp_page_count; e->pages = calloc(e->page_count, sizeof(store_page)); if (e->pages == NULL) { *res = EXTSTORE_INIT_OOM; // FIXME: loop-close. make error label free(e); return NULL; } // interleave the pages between devices f = NULL; // start at the first device. for (i = 0; i < e->page_count; i++) { // find next device with available pages while (1) { // restart the loop if (f == NULL || f->next == NULL) { f = fh; } else { f = f->next; } if (f->page_count) { f->page_count--; break; } } pthread_mutex_init(&e->pages[i].mutex, NULL); e->pages[i].id = i; e->pages[i].fd = f->fd; e->pages[i].free_bucket = f->free_bucket; e->pages[i].offset = f->offset; e->pages[i].free = true; f->offset += e->page_size; } // free page buckets allows the app to organize devices by use case e->free_page_buckets = calloc(cf->page_buckets, sizeof(store_page *)); e->page_bucketcount = cf->page_buckets; for (i = e->page_count-1; i > 0; i--) { e->page_free++; if (e->pages[i].free_bucket == 0) { e->pages[i].next = e->page_freelist; e->page_freelist = &e->pages[i]; } else { int fb = e->pages[i].free_bucket; e->pages[i].next = e->free_page_buckets[fb]; e->free_page_buckets[fb] = &e->pages[i]; } } // 0 is magic "page is freed" version e->version = 1; // scratch data for stats. TODO: malloc failure handle e->stats.page_data = calloc(e->page_count, sizeof(struct extstore_page_data)); e->stats.page_count = e->page_count; e->stats.page_size = e->page_size; // page buckets lazily have pages assigned into them e->page_buckets = calloc(cf->page_buckets, sizeof(store_page *)); e->page_bucketcount = cf->page_buckets; // allocate write buffers // also IO's to use for shipping to IO thread for (i = 0; i < cf->wbuf_count; i++) { _store_wbuf *w = wbuf_new(cf->wbuf_size); obj_io *io = calloc(1, sizeof(obj_io)); /* TODO: on error, loop again and free stack. */ w->next = e->wbuf_stack; e->wbuf_stack = w; io->next = e->io_stack; e->io_stack = io; } pthread_mutex_init(&e->mutex, NULL); pthread_mutex_init(&e->stats_mutex, NULL); e->io_depth = cf->io_depth; // spawn threads e->io_threads = calloc(cf->io_threadcount, sizeof(store_io_thread)); for (i = 0; i < cf->io_threadcount; i++) { pthread_mutex_init(&e->io_threads[i].mutex, NULL); pthread_cond_init(&e->io_threads[i].cond, NULL); e->io_threads[i].e = e; // FIXME: error handling pthread_create(&thread, NULL, extstore_io_thread, &e->io_threads[i]); } e->io_threadcount = cf->io_threadcount; e->maint_thread = calloc(1, sizeof(store_maint_thread)); e->maint_thread->e = e; // FIXME: error handling pthread_mutex_init(&e->maint_thread->mutex, NULL); pthread_cond_init(&e->maint_thread->cond, NULL); pthread_create(&thread, NULL, extstore_maint_thread, e->maint_thread); extstore_run_maint(e); return (void *)e; } void extstore_run_maint(void *ptr) { store_engine *e = (store_engine *)ptr; pthread_cond_signal(&e->maint_thread->cond); } // call with *e locked static store_page *_allocate_page(store_engine *e, unsigned int bucket, unsigned int free_bucket) { assert(!e->page_buckets[bucket] || e->page_buckets[bucket]->allocated == e->page_size); store_page *tmp = NULL; // if a specific free bucket was requested, check there first if (free_bucket != 0 && e->free_page_buckets[free_bucket] != NULL) { assert(e->page_free > 0); tmp = e->free_page_buckets[free_bucket]; e->free_page_buckets[free_bucket] = tmp->next; } // failing that, try the global list. if (tmp == NULL && e->page_freelist != NULL) { tmp = e->page_freelist; e->page_freelist = tmp->next; } E_DEBUG("EXTSTORE: allocating new page\n"); // page_freelist can be empty if the only free pages are specialized and // we didn't just request one. if (e->page_free > 0 && tmp != NULL) { tmp->next = e->page_buckets[bucket]; e->page_buckets[bucket] = tmp; tmp->active = true; tmp->free = false; tmp->closed = false; tmp->version = _next_version(e); tmp->bucket = bucket; e->page_free--; STAT_INCR(e, page_allocs, 1); } else { extstore_run_maint(e); } if (tmp) E_DEBUG("EXTSTORE: got page %u\n", tmp->id); return tmp; } // call with *p locked. locks *e static void _allocate_wbuf(store_engine *e, store_page *p) { _store_wbuf *wbuf = NULL; assert(!p->wbuf); pthread_mutex_lock(&e->mutex); if (e->wbuf_stack) { wbuf = e->wbuf_stack; e->wbuf_stack = wbuf->next; wbuf->next = 0; } pthread_mutex_unlock(&e->mutex); if (wbuf) { wbuf->offset = p->allocated; p->allocated += wbuf->size; wbuf->free = wbuf->size; wbuf->buf_pos = wbuf->buf; wbuf->full = false; wbuf->flushed = false; p->wbuf = wbuf; } } /* callback after wbuf is flushed. can only remove wbuf's from the head onward * if successfully flushed, which complicates this routine. each callback * attempts to free the wbuf stack, which is finally done when the head wbuf's * callback happens. * It's rare flushes would happen out of order. */ static void _wbuf_cb(void *ep, obj_io *io, int ret) { store_engine *e = (store_engine *)ep; store_page *p = &e->pages[io->page_id]; _store_wbuf *w = (_store_wbuf *) io->data; // TODO: Examine return code. Not entirely sure how to handle errors. // Naive first-pass should probably cause the page to close/free. w->flushed = true; pthread_mutex_lock(&p->mutex); assert(p->wbuf != NULL && p->wbuf == w); assert(p->written == w->offset); p->written += w->size; p->wbuf = NULL; if (p->written == e->page_size) p->active = false; // return the wbuf pthread_mutex_lock(&e->mutex); w->next = e->wbuf_stack; e->wbuf_stack = w; // also return the IO we just used. io->next = e->io_stack; e->io_stack = io; pthread_mutex_unlock(&e->mutex); pthread_mutex_unlock(&p->mutex); } /* Wraps pages current wbuf in an io and submits to IO thread. * Called with p locked, locks e. */ static void _submit_wbuf(store_engine *e, store_page *p) { _store_wbuf *w; pthread_mutex_lock(&e->mutex); obj_io *io = e->io_stack; e->io_stack = io->next; pthread_mutex_unlock(&e->mutex); w = p->wbuf; // zero out the end of the wbuf to allow blind readback of data. memset(w->buf + (w->size - w->free), 0, w->free); io->next = NULL; io->mode = OBJ_IO_WRITE; io->page_id = p->id; io->data = w; io->offset = w->offset; io->len = w->size; io->buf = w->buf; io->cb = _wbuf_cb; extstore_submit(e, io); } /* engine write function; takes engine, item_io. * fast fail if no available write buffer (flushing) * lock engine context, find active page, unlock * if page full, submit page/buffer to io thread. * * write is designed to be flaky; if page full, caller must try again to get * new page. best if used from a background thread that can harmlessly retry. */ int extstore_write_request(void *ptr, unsigned int bucket, unsigned int free_bucket, obj_io *io) { store_engine *e = (store_engine *)ptr; store_page *p; int ret = -1; if (bucket >= e->page_bucketcount) return ret; pthread_mutex_lock(&e->mutex); p = e->page_buckets[bucket]; if (!p) { p = _allocate_page(e, bucket, free_bucket); } pthread_mutex_unlock(&e->mutex); if (!p) return ret; pthread_mutex_lock(&p->mutex); // FIXME: can't null out page_buckets!!! // page is full, clear bucket and retry later. if (!p->active || ((!p->wbuf || p->wbuf->full) && p->allocated >= e->page_size)) { pthread_mutex_unlock(&p->mutex); pthread_mutex_lock(&e->mutex); _allocate_page(e, bucket, free_bucket); pthread_mutex_unlock(&e->mutex); return ret; } // if io won't fit, submit IO for wbuf and find new one. if (p->wbuf && p->wbuf->free < io->len && !p->wbuf->full) { _submit_wbuf(e, p); p->wbuf->full = true; } if (!p->wbuf && p->allocated < e->page_size) { _allocate_wbuf(e, p); } // hand over buffer for caller to copy into // leaves p locked. if (p->wbuf && !p->wbuf->full && p->wbuf->free >= io->len) { io->buf = p->wbuf->buf_pos; io->page_id = p->id; return 0; } pthread_mutex_unlock(&p->mutex); // p->written is incremented post-wbuf flush return ret; } /* _must_ be called after a successful write_request. * fills the rest of io structure. */ void extstore_write(void *ptr, obj_io *io) { store_engine *e = (store_engine *)ptr; store_page *p = &e->pages[io->page_id]; io->offset = p->wbuf->offset + (p->wbuf->size - p->wbuf->free); io->page_version = p->version; p->wbuf->buf_pos += io->len; p->wbuf->free -= io->len; p->bytes_used += io->len; p->obj_count++; STAT_L(e); e->stats.bytes_written += io->len; e->stats.bytes_used += io->len; e->stats.objects_written++; e->stats.objects_used++; STAT_UL(e); pthread_mutex_unlock(&p->mutex); } /* engine submit function; takes engine, item_io stack. * lock io_thread context and add stack? * signal io thread to wake. * return success. */ int extstore_submit(void *ptr, obj_io *io) { store_engine *e = (store_engine *)ptr; store_io_thread *t = _get_io_thread(e); pthread_mutex_lock(&t->mutex); if (t->queue == NULL) { t->queue = io; } else { /* Have to put the *io stack at the end of current queue. * FIXME: Optimize by tracking tail. */ obj_io *tmp = t->queue; while (tmp->next != NULL) { tmp = tmp->next; assert(tmp != t->queue); } tmp->next = io; } // TODO: extstore_submit(ptr, io, count) obj_io *tio = io; while (tio != NULL) { t->depth++; tio = tio->next; } pthread_mutex_unlock(&t->mutex); //pthread_mutex_lock(&t->mutex); pthread_cond_signal(&t->cond); //pthread_mutex_unlock(&t->mutex); return 0; } /* engine note delete function: takes engine, page id, size? * note that an item in this page is no longer valid */ int extstore_delete(void *ptr, unsigned int page_id, uint64_t page_version, unsigned int count, unsigned int bytes) { store_engine *e = (store_engine *)ptr; // FIXME: validate page_id in bounds store_page *p = &e->pages[page_id]; int ret = 0; pthread_mutex_lock(&p->mutex); if (!p->closed && p->version == page_version) { if (p->bytes_used >= bytes) { p->bytes_used -= bytes; } else { p->bytes_used = 0; } if (p->obj_count >= count) { p->obj_count -= count; } else { p->obj_count = 0; // caller has bad accounting? } STAT_L(e); e->stats.bytes_used -= bytes; e->stats.objects_used -= count; STAT_UL(e); if (p->obj_count == 0) { extstore_run_maint(e); } } else { ret = -1; } pthread_mutex_unlock(&p->mutex); return ret; } int extstore_check(void *ptr, unsigned int page_id, uint64_t page_version) { store_engine *e = (store_engine *)ptr; store_page *p = &e->pages[page_id]; int ret = 0; pthread_mutex_lock(&p->mutex); if (p->version != page_version) ret = -1; pthread_mutex_unlock(&p->mutex); return ret; } /* allows a compactor to say "we're done with this page, kill it. */ void extstore_close_page(void *ptr, unsigned int page_id, uint64_t page_version) { store_engine *e = (store_engine *)ptr; store_page *p = &e->pages[page_id]; pthread_mutex_lock(&p->mutex); if (!p->closed && p->version == page_version) { p->closed = true; extstore_run_maint(e); } pthread_mutex_unlock(&p->mutex); } /* Finds an attached wbuf that can satisfy the read. * Since wbufs can potentially be flushed to disk out of order, they are only * removed as the head of the list successfully flushes to disk. */ // call with *p locked // FIXME: protect from reading past wbuf static inline int _read_from_wbuf(store_page *p, obj_io *io) { _store_wbuf *wbuf = p->wbuf; assert(wbuf != NULL); assert(io->offset < p->written + wbuf->size); if (io->iov == NULL) { memcpy(io->buf, wbuf->buf + (io->offset - wbuf->offset), io->len); } else { int x; unsigned int off = io->offset - wbuf->offset; // need to loop fill iovecs for (x = 0; x < io->iovcnt; x++) { struct iovec *iov = &io->iov[x]; memcpy(iov->iov_base, wbuf->buf + off, iov->iov_len); off += iov->iov_len; } } return io->len; } /* engine IO thread; takes engine context * manage writes/reads * runs IO callbacks inline after each IO */ // FIXME: protect from reading past page static void *extstore_io_thread(void *arg) { store_io_thread *me = (store_io_thread *)arg; store_engine *e = me->e; while (1) { obj_io *io_stack = NULL; pthread_mutex_lock(&me->mutex); if (me->queue == NULL) { pthread_cond_wait(&me->cond, &me->mutex); } // Pull and disconnect a batch from the queue if (me->queue != NULL) { int i; obj_io *end = NULL; io_stack = me->queue; end = io_stack; for (i = 1; i < e->io_depth; i++) { if (end->next) { end = end->next; } else { break; } } me->depth -= i; me->queue = end->next; end->next = NULL; } pthread_mutex_unlock(&me->mutex); obj_io *cur_io = io_stack; while (cur_io) { // We need to note next before the callback in case the obj_io // gets reused. obj_io *next = cur_io->next; int ret = 0; int do_op = 1; store_page *p = &e->pages[cur_io->page_id]; // TODO: loop if not enough bytes were read/written. switch (cur_io->mode) { case OBJ_IO_READ: // Page is currently open. deal if read is past the end. pthread_mutex_lock(&p->mutex); if (!p->free && !p->closed && p->version == cur_io->page_version) { if (p->active && cur_io->offset >= p->written) { ret = _read_from_wbuf(p, cur_io); do_op = 0; } else { p->refcount++; } STAT_L(e); e->stats.bytes_read += cur_io->len; e->stats.objects_read++; STAT_UL(e); } else { do_op = 0; ret = -2; // TODO: enum in IO for status? } pthread_mutex_unlock(&p->mutex); if (do_op) { #ifdef __APPLE__ ret = lseek(p->fd, SEEK_SET, p->offset + cur_io->offset); if (ret >= 0) { if (cur_io->iov == NULL) { ret = read(p->fd, cur_io->buf, cur_io->len); } else { ret = readv(p->fd, cur_io->iov, cur_io->iovcnt); } } #else if (cur_io->iov == NULL) { ret = pread(p->fd, cur_io->buf, cur_io->len, p->offset + cur_io->offset); } else { ret = preadv(p->fd, cur_io->iov, cur_io->iovcnt, p->offset + cur_io->offset); } #endif } break; case OBJ_IO_WRITE: do_op = 0; // FIXME: Should hold refcount during write. doesn't // currently matter since page can't free while active. ret = pwrite(p->fd, cur_io->buf, cur_io->len, p->offset + cur_io->offset); break; } if (ret == 0) { E_DEBUG("read returned nothing\n"); } #ifdef EXTSTORE_DEBUG if (ret == -1) { perror("read/write op failed"); } #endif cur_io->cb(e, cur_io, ret); if (do_op) { pthread_mutex_lock(&p->mutex); p->refcount--; pthread_mutex_unlock(&p->mutex); } cur_io = next; } } return NULL; } // call with *p locked. static void _free_page(store_engine *e, store_page *p) { store_page *tmp = NULL; store_page *prev = NULL; E_DEBUG("EXTSTORE: freeing page %u\n", p->id); STAT_L(e); e->stats.objects_used -= p->obj_count; e->stats.bytes_used -= p->bytes_used; e->stats.page_reclaims++; STAT_UL(e); pthread_mutex_lock(&e->mutex); // unlink page from bucket list tmp = e->page_buckets[p->bucket]; while (tmp) { if (tmp == p) { if (prev) { prev->next = tmp->next; } else { e->page_buckets[p->bucket] = tmp->next; } tmp->next = NULL; break; } prev = tmp; tmp = tmp->next; } // reset most values p->version = 0; p->obj_count = 0; p->bytes_used = 0; p->allocated = 0; p->written = 0; p->bucket = 0; p->active = false; p->closed = false; p->free = true; // add to page stack // TODO: free_page_buckets first class and remove redundancy? if (p->free_bucket != 0) { p->next = e->free_page_buckets[p->free_bucket]; e->free_page_buckets[p->free_bucket] = p; } else { p->next = e->page_freelist; e->page_freelist = p; } e->page_free++; pthread_mutex_unlock(&e->mutex); } /* engine maint thread; takes engine context. * Uses version to ensure oldest possible objects are being evicted. * Needs interface to inform owner of pages with fewer objects or most space * free, which can then be actively compacted to avoid eviction. * * This gets called asynchronously after every page allocation. Could run less * often if more pages are free. * * Another allocation call is required if an attempted free didn't happen * due to the page having a refcount. */ // TODO: Don't over-evict pages if waiting on refcounts to drop static void *extstore_maint_thread(void *arg) { store_maint_thread *me = (store_maint_thread *)arg; store_engine *e = me->e; struct extstore_page_data *pd = calloc(e->page_count, sizeof(struct extstore_page_data)); pthread_mutex_lock(&me->mutex); while (1) { int i; bool do_evict = false; unsigned int low_page = 0; uint64_t low_version = ULLONG_MAX; pthread_cond_wait(&me->cond, &me->mutex); pthread_mutex_lock(&e->mutex); // default freelist requires at least one page free. // specialized freelists fall back to default once full. if (e->page_free == 0 || e->page_freelist == NULL) { do_evict = true; } pthread_mutex_unlock(&e->mutex); memset(pd, 0, sizeof(struct extstore_page_data) * e->page_count); for (i = 0; i < e->page_count; i++) { store_page *p = &e->pages[i]; pthread_mutex_lock(&p->mutex); pd[p->id].free_bucket = p->free_bucket; if (p->active || p->free) { pthread_mutex_unlock(&p->mutex); continue; } if (p->obj_count > 0 && !p->closed) { pd[p->id].version = p->version; pd[p->id].bytes_used = p->bytes_used; pd[p->id].bucket = p->bucket; // low_version/low_page are only used in the eviction // scenario. when we evict, it's only to fill the default page // bucket again. // TODO: experiment with allowing evicting up to a single page // for any specific free bucket. this is *probably* required // since it could cause a load bias on default-only devices? if (p->free_bucket == 0 && p->version < low_version) { low_version = p->version; low_page = i; } } if ((p->obj_count == 0 || p->closed) && p->refcount == 0) { _free_page(e, p); // Found a page to free, no longer need to evict. do_evict = false; } pthread_mutex_unlock(&p->mutex); } if (do_evict && low_version != ULLONG_MAX) { store_page *p = &e->pages[low_page]; E_DEBUG("EXTSTORE: evicting page [%d] [v: %llu]\n", p->id, (unsigned long long) p->version); pthread_mutex_lock(&p->mutex); if (!p->closed) { p->closed = true; STAT_L(e); e->stats.page_evictions++; e->stats.objects_evicted += p->obj_count; e->stats.bytes_evicted += p->bytes_used; STAT_UL(e); if (p->refcount == 0) { _free_page(e, p); } } pthread_mutex_unlock(&p->mutex); } // copy the page data into engine context so callers can use it from // the stats lock. STAT_L(e); memcpy(e->stats.page_data, pd, sizeof(struct extstore_page_data) * e->page_count); STAT_UL(e); } return NULL; } memcached-1.5.22/crawler.c0000664000175000017500000005500013606212426012237 00000000000000/* Copyright 2016 Netflix. * * Use and distribution licensed under the BSD license. See * the LICENSE file for full text. */ /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #include "memcached.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define LARGEST_ID POWER_LARGEST typedef struct { void *c; /* original connection structure. still with source thread attached. */ int sfd; /* client fd. */ bipbuf_t *buf; /* output buffer */ char *cbuf; /* current buffer */ } crawler_client_t; typedef struct _crawler_module_t crawler_module_t; typedef void (*crawler_eval_func)(crawler_module_t *cm, item *it, uint32_t hv, int slab_cls); typedef int (*crawler_init_func)(crawler_module_t *cm, void *data); // TODO: init args? typedef void (*crawler_deinit_func)(crawler_module_t *cm); // TODO: extra args? typedef void (*crawler_doneclass_func)(crawler_module_t *cm, int slab_cls); typedef void (*crawler_finalize_func)(crawler_module_t *cm); typedef struct { crawler_init_func init; /* run before crawl starts */ crawler_eval_func eval; /* runs on an item. */ crawler_doneclass_func doneclass; /* runs once per sub-crawler completion. */ crawler_finalize_func finalize; /* runs once when all sub-crawlers are done. */ bool needs_lock; /* whether or not we need the LRU lock held when eval is called */ bool needs_client; /* whether or not to grab onto the remote client */ } crawler_module_reg_t; struct _crawler_module_t { void *data; /* opaque data pointer */ crawler_client_t c; crawler_module_reg_t *mod; }; static int crawler_expired_init(crawler_module_t *cm, void *data); static void crawler_expired_doneclass(crawler_module_t *cm, int slab_cls); static void crawler_expired_finalize(crawler_module_t *cm); static void crawler_expired_eval(crawler_module_t *cm, item *search, uint32_t hv, int i); crawler_module_reg_t crawler_expired_mod = { .init = crawler_expired_init, .eval = crawler_expired_eval, .doneclass = crawler_expired_doneclass, .finalize = crawler_expired_finalize, .needs_lock = true, .needs_client = false }; static void crawler_metadump_eval(crawler_module_t *cm, item *search, uint32_t hv, int i); static void crawler_metadump_finalize(crawler_module_t *cm); crawler_module_reg_t crawler_metadump_mod = { .init = NULL, .eval = crawler_metadump_eval, .doneclass = NULL, .finalize = crawler_metadump_finalize, .needs_lock = false, .needs_client = true }; crawler_module_reg_t *crawler_mod_regs[3] = { &crawler_expired_mod, &crawler_expired_mod, &crawler_metadump_mod }; static int lru_crawler_client_getbuf(crawler_client_t *c); crawler_module_t active_crawler_mod; enum crawler_run_type active_crawler_type; static crawler crawlers[LARGEST_ID]; static int crawler_count = 0; static volatile int do_run_lru_crawler_thread = 0; static int lru_crawler_initialized = 0; static pthread_mutex_t lru_crawler_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t lru_crawler_cond = PTHREAD_COND_INITIALIZER; #ifdef EXTSTORE /* TODO: pass this around */ static void *storage; #endif /* Will crawl all slab classes a minimum of once per hour */ #define MAX_MAINTCRAWL_WAIT 60 * 60 /*** LRU CRAWLER THREAD ***/ #define LRU_CRAWLER_WRITEBUF 8192 static void lru_crawler_close_client(crawler_client_t *c) { //fprintf(stderr, "CRAWLER: Closing client\n"); sidethread_conn_close(c->c); c->c = NULL; c->cbuf = NULL; bipbuf_free(c->buf); c->buf = NULL; } static void lru_crawler_release_client(crawler_client_t *c) { //fprintf(stderr, "CRAWLER: Closing client\n"); redispatch_conn(c->c); c->c = NULL; c->cbuf = NULL; bipbuf_free(c->buf); c->buf = NULL; } static int crawler_expired_init(crawler_module_t *cm, void *data) { struct crawler_expired_data *d; if (data != NULL) { d = data; d->is_external = true; cm->data = data; } else { // allocate data. d = calloc(1, sizeof(struct crawler_expired_data)); if (d == NULL) { return -1; } // init lock. pthread_mutex_init(&d->lock, NULL); d->is_external = false; d->start_time = current_time; cm->data = d; } pthread_mutex_lock(&d->lock); memset(&d->crawlerstats, 0, sizeof(crawlerstats_t) * POWER_LARGEST); for (int x = 0; x < POWER_LARGEST; x++) { d->crawlerstats[x].start_time = current_time; d->crawlerstats[x].run_complete = false; } pthread_mutex_unlock(&d->lock); return 0; } static void crawler_expired_doneclass(crawler_module_t *cm, int slab_cls) { struct crawler_expired_data *d = (struct crawler_expired_data *) cm->data; pthread_mutex_lock(&d->lock); d->crawlerstats[slab_cls].end_time = current_time; d->crawlerstats[slab_cls].run_complete = true; pthread_mutex_unlock(&d->lock); } static void crawler_expired_finalize(crawler_module_t *cm) { struct crawler_expired_data *d = (struct crawler_expired_data *) cm->data; pthread_mutex_lock(&d->lock); d->end_time = current_time; d->crawl_complete = true; pthread_mutex_unlock(&d->lock); if (!d->is_external) { free(d); } } /* I pulled this out to make the main thread clearer, but it reaches into the * main thread's values too much. Should rethink again. */ static void crawler_expired_eval(crawler_module_t *cm, item *search, uint32_t hv, int i) { struct crawler_expired_data *d = (struct crawler_expired_data *) cm->data; pthread_mutex_lock(&d->lock); crawlerstats_t *s = &d->crawlerstats[i]; int is_flushed = item_is_flushed(search); #ifdef EXTSTORE bool is_valid = true; if (search->it_flags & ITEM_HDR) { item_hdr *hdr = (item_hdr *)ITEM_data(search); if (extstore_check(storage, hdr->page_id, hdr->page_version) != 0) is_valid = false; } #endif if ((search->exptime != 0 && search->exptime < current_time) || is_flushed #ifdef EXTSTORE || !is_valid #endif ) { crawlers[i].reclaimed++; s->reclaimed++; if (settings.verbose > 1) { int ii; char *key = ITEM_key(search); fprintf(stderr, "LRU crawler found an expired item (flags: %d, slab: %d): ", search->it_flags, search->slabs_clsid); for (ii = 0; ii < search->nkey; ++ii) { fprintf(stderr, "%c", key[ii]); } fprintf(stderr, "\n"); } if ((search->it_flags & ITEM_FETCHED) == 0 && !is_flushed) { crawlers[i].unfetched++; } #ifdef EXTSTORE STORAGE_delete(storage, search); #endif do_item_unlink_nolock(search, hv); do_item_remove(search); } else { s->seen++; refcount_decr(search); if (search->exptime == 0) { s->noexp++; } else if (search->exptime - current_time > 3599) { s->ttl_hourplus++; } else { rel_time_t ttl_remain = search->exptime - current_time; int bucket = ttl_remain / 60; if (bucket <= 60) { s->histo[bucket]++; } } } pthread_mutex_unlock(&d->lock); } static void crawler_metadump_eval(crawler_module_t *cm, item *it, uint32_t hv, int i) { //int slab_id = CLEAR_LRU(i); char keybuf[KEY_MAX_URI_ENCODED_LENGTH]; int is_flushed = item_is_flushed(it); /* Ignore expired content. */ if ((it->exptime != 0 && it->exptime < current_time) || is_flushed) { refcount_decr(it); return; } // TODO: uriencode directly into the buffer. uriencode(ITEM_key(it), keybuf, it->nkey, KEY_MAX_URI_ENCODED_LENGTH); int total = snprintf(cm->c.cbuf, 4096, "key=%s exp=%ld la=%llu cas=%llu fetch=%s cls=%u size=%lu\n", keybuf, (it->exptime == 0) ? -1 : (long)(it->exptime + process_started), (unsigned long long)(it->time + process_started), (unsigned long long)ITEM_get_cas(it), (it->it_flags & ITEM_FETCHED) ? "yes" : "no", ITEM_clsid(it), (unsigned long) ITEM_ntotal(it)); refcount_decr(it); // TODO: some way of tracking the errors. these are very unlikely though. if (total >= LRU_CRAWLER_WRITEBUF - 1 || total <= 0) { /* Failed to write, don't push it. */ return; } bipbuf_push(cm->c.buf, total); } static void crawler_metadump_finalize(crawler_module_t *cm) { if (cm->c.c != NULL) { // Ensure space for final message. lru_crawler_client_getbuf(&cm->c); memcpy(cm->c.cbuf, "END\r\n", 5); bipbuf_push(cm->c.buf, 5); } } static int lru_crawler_poll(crawler_client_t *c) { unsigned char *data; unsigned int data_size = 0; struct pollfd to_poll[1]; to_poll[0].fd = c->sfd; to_poll[0].events = POLLOUT; int ret = poll(to_poll, 1, 1000); if (ret < 0) { // fatal. return -1; } if (ret == 0) return 0; if (to_poll[0].revents & POLLIN) { char buf[1]; int res = ((conn*)c->c)->read(c->c, buf, 1); if (res == 0 || (res == -1 && (errno != EAGAIN && errno != EWOULDBLOCK))) { lru_crawler_close_client(c); return -1; } } if ((data = bipbuf_peek_all(c->buf, &data_size)) != NULL) { if (to_poll[0].revents & (POLLHUP|POLLERR)) { lru_crawler_close_client(c); return -1; } else if (to_poll[0].revents & POLLOUT) { int total = ((conn*)c->c)->write(c->c, data, data_size); if (total == -1) { if (errno != EAGAIN && errno != EWOULDBLOCK) { lru_crawler_close_client(c); return -1; } } else if (total == 0) { lru_crawler_close_client(c); return -1; } else { bipbuf_poll(c->buf, total); } } } return 0; } /* Grab some space to work with, if none exists, run the poll() loop and wait * for it to clear up or close. * Return NULL if closed. */ static int lru_crawler_client_getbuf(crawler_client_t *c) { void *buf = NULL; if (c->c == NULL) return -1; /* not enough space. */ while ((buf = bipbuf_request(c->buf, LRU_CRAWLER_WRITEBUF)) == NULL) { // TODO: max loops before closing. int ret = lru_crawler_poll(c); if (ret < 0) return ret; } c->cbuf = buf; return 0; } static void lru_crawler_class_done(int i) { crawlers[i].it_flags = 0; crawler_count--; do_item_unlinktail_q((item *)&crawlers[i]); do_item_stats_add_crawl(i, crawlers[i].reclaimed, crawlers[i].unfetched, crawlers[i].checked); pthread_mutex_unlock(&lru_locks[i]); if (active_crawler_mod.mod->doneclass != NULL) active_crawler_mod.mod->doneclass(&active_crawler_mod, i); } static void *item_crawler_thread(void *arg) { int i; int crawls_persleep = settings.crawls_persleep; pthread_mutex_lock(&lru_crawler_lock); pthread_cond_signal(&lru_crawler_cond); settings.lru_crawler = true; if (settings.verbose > 2) fprintf(stderr, "Starting LRU crawler background thread\n"); while (do_run_lru_crawler_thread) { pthread_cond_wait(&lru_crawler_cond, &lru_crawler_lock); while (crawler_count) { item *search = NULL; void *hold_lock = NULL; for (i = POWER_SMALLEST; i < LARGEST_ID; i++) { if (crawlers[i].it_flags != 1) { continue; } /* Get memory from bipbuf, if client has no space, flush. */ if (active_crawler_mod.c.c != NULL) { int ret = lru_crawler_client_getbuf(&active_crawler_mod.c); if (ret != 0) { lru_crawler_class_done(i); continue; } } else if (active_crawler_mod.mod->needs_client) { lru_crawler_class_done(i); continue; } pthread_mutex_lock(&lru_locks[i]); search = do_item_crawl_q((item *)&crawlers[i]); if (search == NULL || (crawlers[i].remaining && --crawlers[i].remaining < 1)) { if (settings.verbose > 2) fprintf(stderr, "Nothing left to crawl for %d\n", i); lru_crawler_class_done(i); continue; } uint32_t hv = hash(ITEM_key(search), search->nkey); /* Attempt to hash item lock the "search" item. If locked, no * other callers can incr the refcount */ if ((hold_lock = item_trylock(hv)) == NULL) { pthread_mutex_unlock(&lru_locks[i]); continue; } /* Now see if the item is refcount locked */ if (refcount_incr(search) != 2) { refcount_decr(search); if (hold_lock) item_trylock_unlock(hold_lock); pthread_mutex_unlock(&lru_locks[i]); continue; } crawlers[i].checked++; /* Frees the item or decrements the refcount. */ /* Interface for this could improve: do the free/decr here * instead? */ if (!active_crawler_mod.mod->needs_lock) { pthread_mutex_unlock(&lru_locks[i]); } active_crawler_mod.mod->eval(&active_crawler_mod, search, hv, i); if (hold_lock) item_trylock_unlock(hold_lock); if (active_crawler_mod.mod->needs_lock) { pthread_mutex_unlock(&lru_locks[i]); } if (crawls_persleep-- <= 0 && settings.lru_crawler_sleep) { pthread_mutex_unlock(&lru_crawler_lock); usleep(settings.lru_crawler_sleep); pthread_mutex_lock(&lru_crawler_lock); crawls_persleep = settings.crawls_persleep; } else if (!settings.lru_crawler_sleep) { // TODO: only cycle lock every N? pthread_mutex_unlock(&lru_crawler_lock); pthread_mutex_lock(&lru_crawler_lock); } } } if (active_crawler_mod.mod != NULL) { if (active_crawler_mod.mod->finalize != NULL) active_crawler_mod.mod->finalize(&active_crawler_mod); while (active_crawler_mod.c.c != NULL && bipbuf_used(active_crawler_mod.c.buf)) { lru_crawler_poll(&active_crawler_mod.c); } // Double checking in case the client closed during the poll if (active_crawler_mod.c.c != NULL) { lru_crawler_release_client(&active_crawler_mod.c); } active_crawler_mod.mod = NULL; } if (settings.verbose > 2) fprintf(stderr, "LRU crawler thread sleeping\n"); STATS_LOCK(); stats_state.lru_crawler_running = false; STATS_UNLOCK(); } pthread_mutex_unlock(&lru_crawler_lock); if (settings.verbose > 2) fprintf(stderr, "LRU crawler thread stopping\n"); settings.lru_crawler = false; return NULL; } static pthread_t item_crawler_tid; int stop_item_crawler_thread(bool wait) { int ret; pthread_mutex_lock(&lru_crawler_lock); if (do_run_lru_crawler_thread == 0) { pthread_mutex_unlock(&lru_crawler_lock); return 0; } do_run_lru_crawler_thread = 0; pthread_cond_signal(&lru_crawler_cond); pthread_mutex_unlock(&lru_crawler_lock); if (wait && (ret = pthread_join(item_crawler_tid, NULL)) != 0) { fprintf(stderr, "Failed to stop LRU crawler thread: %s\n", strerror(ret)); return -1; } return 0; } /* Lock dance to "block" until thread is waiting on its condition: * caller locks mtx. caller spawns thread. * thread blocks on mutex. * caller waits on condition, releases lock. * thread gets lock, sends signal. * caller can't wait, as thread has lock. * thread waits on condition, releases lock * caller wakes on condition, gets lock. * caller immediately releases lock. * thread is now safely waiting on condition before the caller returns. */ int start_item_crawler_thread(void) { int ret; if (settings.lru_crawler) return -1; pthread_mutex_lock(&lru_crawler_lock); do_run_lru_crawler_thread = 1; if ((ret = pthread_create(&item_crawler_tid, NULL, item_crawler_thread, NULL)) != 0) { fprintf(stderr, "Can't create LRU crawler thread: %s\n", strerror(ret)); pthread_mutex_unlock(&lru_crawler_lock); return -1; } /* Avoid returning until the crawler has actually started */ pthread_cond_wait(&lru_crawler_cond, &lru_crawler_lock); pthread_mutex_unlock(&lru_crawler_lock); return 0; } /* 'remaining' is passed in so the LRU maintainer thread can scrub the whole * LRU every time. */ static int do_lru_crawler_start(uint32_t id, uint32_t remaining) { uint32_t sid = id; int starts = 0; pthread_mutex_lock(&lru_locks[sid]); if (crawlers[sid].it_flags == 0) { if (settings.verbose > 2) fprintf(stderr, "Kicking LRU crawler off for LRU %u\n", sid); crawlers[sid].nbytes = 0; crawlers[sid].nkey = 0; crawlers[sid].it_flags = 1; /* For a crawler, this means enabled. */ crawlers[sid].next = 0; crawlers[sid].prev = 0; crawlers[sid].time = 0; if (remaining == LRU_CRAWLER_CAP_REMAINING) { remaining = do_get_lru_size(sid); } /* Values for remaining: * remaining = 0 * - scan all elements, until a NULL is reached * - if empty, NULL is reached right away * remaining = n + 1 * - first n elements are parsed (or until a NULL is reached) */ if (remaining) remaining++; crawlers[sid].remaining = remaining; crawlers[sid].slabs_clsid = sid; crawlers[sid].reclaimed = 0; crawlers[sid].unfetched = 0; crawlers[sid].checked = 0; do_item_linktail_q((item *)&crawlers[sid]); crawler_count++; starts++; } pthread_mutex_unlock(&lru_locks[sid]); if (starts) { STATS_LOCK(); stats_state.lru_crawler_running = true; stats.lru_crawler_starts++; STATS_UNLOCK(); } return starts; } static int lru_crawler_set_client(crawler_module_t *cm, void *c, const int sfd) { crawler_client_t *crawlc = &cm->c; if (crawlc->c != NULL) { return -1; } crawlc->c = c; crawlc->sfd = sfd; crawlc->buf = bipbuf_new(1024 * 128); if (crawlc->buf == NULL) { return -2; } return 0; } int lru_crawler_start(uint8_t *ids, uint32_t remaining, const enum crawler_run_type type, void *data, void *c, const int sfd) { int starts = 0; bool is_running; static rel_time_t block_ae_until = 0; pthread_mutex_lock(&lru_crawler_lock); STATS_LOCK(); is_running = stats_state.lru_crawler_running; STATS_UNLOCK(); if (do_run_lru_crawler_thread == 0) { pthread_mutex_unlock(&lru_crawler_lock); return -2; } if (is_running && !(type == CRAWLER_AUTOEXPIRE && active_crawler_type == CRAWLER_AUTOEXPIRE)) { pthread_mutex_unlock(&lru_crawler_lock); block_ae_until = current_time + 60; return -1; } if (type == CRAWLER_AUTOEXPIRE && block_ae_until > current_time) { pthread_mutex_unlock(&lru_crawler_lock); return -1; } /* Configure the module */ if (!is_running) { assert(crawler_mod_regs[type] != NULL); active_crawler_mod.mod = crawler_mod_regs[type]; active_crawler_type = type; if (active_crawler_mod.mod->init != NULL) { active_crawler_mod.mod->init(&active_crawler_mod, data); } if (active_crawler_mod.mod->needs_client) { if (c == NULL || sfd == 0) { pthread_mutex_unlock(&lru_crawler_lock); return -2; } if (lru_crawler_set_client(&active_crawler_mod, c, sfd) != 0) { pthread_mutex_unlock(&lru_crawler_lock); return -2; } } } /* we allow the autocrawler to restart sub-LRU's before completion */ for (int sid = POWER_SMALLEST; sid < POWER_LARGEST; sid++) { if (ids[sid]) starts += do_lru_crawler_start(sid, remaining); } if (starts) { pthread_cond_signal(&lru_crawler_cond); } pthread_mutex_unlock(&lru_crawler_lock); return starts; } /* * Also only clear the crawlerstats once per sid. */ enum crawler_result_type lru_crawler_crawl(char *slabs, const enum crawler_run_type type, void *c, const int sfd, unsigned int remaining) { char *b = NULL; uint32_t sid = 0; int starts = 0; uint8_t tocrawl[POWER_LARGEST]; /* FIXME: I added this while debugging. Don't think it's needed? */ memset(tocrawl, 0, sizeof(uint8_t) * POWER_LARGEST); if (strcmp(slabs, "all") == 0) { for (sid = 0; sid < POWER_LARGEST; sid++) { tocrawl[sid] = 1; } } else { for (char *p = strtok_r(slabs, ",", &b); p != NULL; p = strtok_r(NULL, ",", &b)) { if (!safe_strtoul(p, &sid) || sid < POWER_SMALLEST || sid >= MAX_NUMBER_OF_SLAB_CLASSES) { pthread_mutex_unlock(&lru_crawler_lock); return CRAWLER_BADCLASS; } tocrawl[sid | TEMP_LRU] = 1; tocrawl[sid | HOT_LRU] = 1; tocrawl[sid | WARM_LRU] = 1; tocrawl[sid | COLD_LRU] = 1; } } starts = lru_crawler_start(tocrawl, remaining, type, NULL, c, sfd); if (starts == -1) { return CRAWLER_RUNNING; } else if (starts == -2) { return CRAWLER_ERROR; /* FIXME: not very helpful. */ } else if (starts) { return CRAWLER_OK; } else { return CRAWLER_NOTSTARTED; } } /* If we hold this lock, crawler can't wake up or move */ void lru_crawler_pause(void) { pthread_mutex_lock(&lru_crawler_lock); } void lru_crawler_resume(void) { pthread_mutex_unlock(&lru_crawler_lock); } int init_lru_crawler(void *arg) { if (lru_crawler_initialized == 0) { #ifdef EXTSTORE storage = arg; #endif active_crawler_mod.c.c = NULL; active_crawler_mod.mod = NULL; active_crawler_mod.data = NULL; lru_crawler_initialized = 1; } return 0; } memcached-1.5.22/configure.ac0000664000175000017500000005604213611703161012726 00000000000000AC_PREREQ(2.52) m4_include([version.m4]) m4_include([m4/c99-backport.m4]) AC_INIT([memcached], [VERSION_NUMBER], [memcached@googlegroups.com]) AC_CANONICAL_HOST AC_CONFIG_SRCDIR([memcached.c]) AM_INIT_AUTOMAKE([foreign]) AM_CONFIG_HEADER([config.h]) AC_PROG_CC dnl ********************************************************************** dnl DETECT_ICC ([ACTION-IF-YES], [ACTION-IF-NO]) dnl dnl check if this is the Intel ICC compiler, and if so run the ACTION-IF-YES dnl sets the $ICC variable to "yes" or "no" dnl ********************************************************************** AC_DEFUN([DETECT_ICC], [ ICC="no" AC_MSG_CHECKING([for icc in use]) if test "$GCC" = "yes"; then dnl check if this is icc acting as gcc in disguise AC_EGREP_CPP([^__INTEL_COMPILER], [__INTEL_COMPILER], AC_MSG_RESULT([no]) [$2], AC_MSG_RESULT([yes]) [$1] ICC="yes") else AC_MSG_RESULT([no]) [$2] fi ]) DETECT_ICC([], []) dnl ********************************************************************** dnl DETECT_CLANG ([ACTION-IF-YES], [ACTION-IF-NO]) dnl dnl check if compiler is clang, and if so run the ACTION-IF-YES sets the dnl $CLANG variable to "yes" or "no" dnl ********************************************************************** AC_DEFUN([DETECT_CLANG], [ AC_MSG_CHECKING([for clang in use]) AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([], [[ #ifndef __clang__ not clang #endif ]])], [CLANG=yes], [CLANG=no]) AC_MSG_RESULT([$CLANG]) AS_IF([test "$CLANG" = "yes"],[$1],[$2]) ]) DETECT_CLANG([],[]) dnl ********************************************************************** dnl DETECT_SUNCC ([ACTION-IF-YES], [ACTION-IF-NO]) dnl dnl check if this is the Sun Studio compiler, and if so run the ACTION-IF-YES dnl sets the $SUNCC variable to "yes" or "no" dnl ********************************************************************** AC_DEFUN([DETECT_SUNCC], [ AC_CHECK_DECL([__SUNPRO_C], [SUNCC="yes"], [SUNCC="no"]) AS_IF(test "x$SUNCC" = "xyes", [$1], [$2]) ]) DETECT_SUNCC([CFLAGS="-mt $CFLAGS"], []) AS_IF([test "$ICC" = "yes" -o "$GCC" = "yes"], [ AS_IF(test "$CLANG" = "no",[CFLAGS="$CFLAGS -pthread"]) ]) if test "$ICC" = "no"; then AC_PROG_CC_C99 fi AM_PROG_CC_C_O AC_PROG_INSTALL AC_ARG_ENABLE(arm_crc32, [AS_HELP_STRING([--enable-arm-crc32], [Enable ARMv8 CRC32 instructions])]) AC_ARG_ENABLE(extstore, [AS_HELP_STRING([--enable-extstore], [Enable external storage EXPERIMENTAL ])]) AC_ARG_ENABLE(seccomp, [AS_HELP_STRING([--enable-seccomp],[Enable seccomp restrictions EXPERIMENTAL])]) AC_ARG_ENABLE(sasl, [AS_HELP_STRING([--enable-sasl],[Enable SASL authentication])]) AC_ARG_ENABLE(sasl_pwdb, [AS_HELP_STRING([--enable-sasl-pwdb],[Enable plaintext password db])]) AS_IF([test "x$enable_sasl_pwdb" = "xyes"], [enable_sasl=yes ]) AC_ARG_ENABLE(tls, [AS_HELP_STRING([--enable-tls], [Enable Transport Layer Security EXPERIMENTAL ])]) AC_ARG_ENABLE(asan, [AS_HELP_STRING([--enable-asan], [Compile with ASAN EXPERIMENTAL ])]) dnl ********************************************************************** dnl DETECT_SASL_CB_GETCONF dnl dnl check if we can use SASL_CB_GETCONF dnl ********************************************************************** AC_DEFUN([AC_C_DETECT_SASL_CB_GETCONF], [ AC_CACHE_CHECK([for SASL_CB_GETCONF], [ac_cv_c_sasl_cb_getconf], [AC_TRY_COMPILE( [ #include ], [ unsigned long val = SASL_CB_GETCONF; ], [ ac_cv_c_sasl_cb_getconf=yes ], [ ac_cv_c_sasl_cb_getconf=no ]) ]) AS_IF([test "$ac_cv_c_sasl_cb_getconf" = "yes"], [AC_DEFINE([HAVE_SASL_CB_GETCONF], 1, [Set to nonzero if your SASL implementation supports SASL_CB_GETCONF])]) ]) dnl ********************************************************************** dnl DETECT_SASL_CB_GETCONFPATH dnl dnl check if we can use SASL_CB_GETCONFPATH dnl ********************************************************************** AC_DEFUN([AC_C_DETECT_SASL_CB_GETCONFPATH], [ AC_CACHE_CHECK([for SASL_CB_GETCONFPATH], [ac_cv_c_sasl_cb_getconfpath], [AC_TRY_COMPILE( [ #include ], [ unsigned long val = SASL_CB_GETCONFPATH; ], [ ac_cv_c_sasl_cb_getconfpath=yes ], [ ac_cv_c_sasl_cb_getconfpath=no ]) ]) AS_IF([test "$ac_cv_c_sasl_cb_getconfpath" = "yes"], [AC_DEFINE([HAVE_SASL_CB_GETCONFPATH], 1, [Set to nonzero if your SASL implementation supports SASL_CB_GETCONFPATH])]) ]) AC_CHECK_HEADERS([sasl/sasl.h]) if test "x$enable_sasl" = "xyes"; then AC_C_DETECT_SASL_CB_GETCONF AC_C_DETECT_SASL_CB_GETCONFPATH AC_DEFINE([ENABLE_SASL],1,[Set to nonzero if you want to include SASL]) AC_SEARCH_LIBS([sasl_server_init], [sasl2 sasl], [], [ AC_MSG_ERROR([Failed to locate the library containing sasl_server_init]) ]) AS_IF([test "x$enable_sasl_pwdb" = "xyes"], [AC_DEFINE([ENABLE_SASL_PWDB], 1, [Set to nonzero if you want to enable a SASL pwdb])]) fi AC_ARG_ENABLE(dtrace, [AS_HELP_STRING([--enable-dtrace],[Enable dtrace probes])]) if test "x$enable_dtrace" = "xyes"; then AC_PATH_PROG([DTRACE], [dtrace], "no", [/usr/sbin:$PATH]) if test "x$DTRACE" != "xno"; then AC_DEFINE([ENABLE_DTRACE],1,[Set to nonzero if you want to include DTRACE]) build_dtrace=yes $DTRACE -h -o conftest.h -s memcached_dtrace.d 2>/dev/zero if test $? -eq 0 then dtrace_instrument_obj=yes rm conftest.h fi if test "`which tr`" = "/usr/ucb/tr"; then AC_MSG_ERROR([Please remove /usr/ucb from your path. See man standards for more info]) fi else AC_MSG_ERROR([Need dtrace binary and OS support.]) fi fi if test "x$enable_extstore" = "xyes"; then AC_DEFINE([EXTSTORE],1,[Set to nonzero if you want to enable extstore]) fi if test "x$enable_tls" = "xyes"; then AC_DEFINE([TLS],1,[Set to nonzero if you want to enable TLS]) fi if test "x$enable_arm_crc32" = "xyes"; then AC_DEFINE([ARM_CRC32],1,[Set to nonzero if you want to enable ARMv8 crc32]) fi if test "x$enable_asan" = "xyes"; then AC_DEFINE([ASAN],1,[Set to nonzero if you want to compile using ASAN]) fi AM_CONDITIONAL([BUILD_DTRACE],[test "$build_dtrace" = "yes"]) AM_CONDITIONAL([DTRACE_INSTRUMENT_OBJ],[test "$dtrace_instrument_obj" = "yes"]) AM_CONDITIONAL([ENABLE_SASL],[test "$enable_sasl" = "yes"]) AM_CONDITIONAL([ENABLE_EXTSTORE],[test "$enable_extstore" = "yes"]) AM_CONDITIONAL([ENABLE_ARM_CRC32],[test "$enable_arm_crc32" = "yes"]) AM_CONDITIONAL([ENABLE_TLS],[test "$enable_tls" = "yes"]) AM_CONDITIONAL([ENABLE_ASAN],[test "$enable_asan" = "yes"]) AC_SUBST(DTRACE) AC_SUBST(DTRACEFLAGS) AC_SUBST(ENABLE_SASL) AC_SUBST(PROFILER_LDFLAGS) AC_ARG_ENABLE(coverage, [AS_HELP_STRING([--disable-coverage],[Disable code coverage])]) if test "x$enable_coverage" != "xno"; then if test "$GCC" = "yes" -a "$ICC" != "yes" -a "$CLANG" != "yes" then CFLAGS="$CFLAGS -pthread" AC_PATH_PROG([PROFILER], [gcov], "no", [$PATH]) if test "x$PROFILER" != "xno"; then # Issue 97: The existence of gcov doesn't mean we have -lgcov AC_CHECK_LIB(gcov, main, [ PROFILER_FLAGS="-fprofile-arcs -ftest-coverage" PROFILER_LDFLAGS="-lgcov" ], [ PROFILER_FLAGS= PROFILER_LDFLAGS= ]) fi elif test "$SUNCC" = "yes" then AC_PATH_PROG([PROFILER], [tcov], "no", [$PATH]) if test "x$PROFILER" != "xno"; then PROFILER_FLAGS=-xprofile=tcov fi elif test "x$CLANG" != "xno" then AC_PATH_PROG([PROFILER], [gcov], "no", [$PATH]) if test "x$PROFILER" != "xno" then PROFILER_FLAGS="-fprofile-arcs -ftest-coverage" PROFILER_LDFLAGS= fi fi fi AC_SUBST(PROFILER_FLAGS) AC_ARG_ENABLE(64bit, [AS_HELP_STRING([--enable-64bit],[build 64bit version])]) if test "x$enable_64bit" = "xyes" then org_cflags=$CFLAGS CFLAGS=-m64 AC_RUN_IFELSE( [AC_LANG_PROGRAM([], [dnl return sizeof(void*) == 8 ? 0 : 1; ]) ],[ CFLAGS="-m64 $org_cflags" ],[ AC_MSG_ERROR([Don't know how to build a 64-bit object.]) ],[ dnl cross compile AC_MSG_WARN([Assuming no extra CFLAGS are required for cross-compiling 64bit version.]) ]) fi dnl Check if data pointer is 64bit or not AC_CHECK_SIZEOF([void *]) # Issue 213: Search for clock_gettime to help people linking # with a static version of libevent AC_SEARCH_LIBS(clock_gettime, rt) # Issue 214: Search for the network libraries _before_ searching # for libevent (to help people linking with static libevent) AC_SEARCH_LIBS(socket, socket) AC_SEARCH_LIBS(gethostbyname, nsl) trylibeventdir="" AC_ARG_WITH(libevent, [ --with-libevent=PATH Specify path to libevent installation ], [ if test "x$withval" != "xno" ; then trylibeventdir=$withval fi ] ) dnl ------------------------------------------------------ dnl libevent detection. swiped from Tor. modified a bit. LIBEVENT_URL=https://www.monkey.org/~provos/libevent/ AC_CACHE_CHECK([for libevent directory], ac_cv_libevent_dir, [ saved_LIBS="$LIBS" saved_LDFLAGS="$LDFLAGS" saved_CPPFLAGS="$CPPFLAGS" le_found=no for ledir in $trylibeventdir "" $prefix /usr/local ; do LDFLAGS="$saved_LDFLAGS" LIBS="-levent $saved_LIBS" # Skip the directory if it isn't there. if test ! -z "$ledir" -a ! -d "$ledir" ; then continue; fi if test ! -z "$ledir" ; then if test -d "$ledir/lib" ; then LDFLAGS="-L$ledir/lib $LDFLAGS" else LDFLAGS="-L$ledir $LDFLAGS" fi if test -d "$ledir/include" ; then CPPFLAGS="-I$ledir/include $CPPFLAGS" else CPPFLAGS="-I$ledir $CPPFLAGS" fi fi # Can I compile and link it? AC_TRY_LINK([#include #include #include ], [ event_init(); ], [ libevent_linked=yes ], [ libevent_linked=no ]) if test $libevent_linked = yes; then if test ! -z "$ledir" ; then ac_cv_libevent_dir=$ledir _myos=`echo $target_os | cut -f 1 -d .` AS_IF(test "$SUNCC" = "yes" -o "x$_myos" = "xsolaris2", [saved_LDFLAGS="$saved_LDFLAGS -Wl,-R$ledir/lib"], [AS_IF(test "$GCC" = "yes", [saved_LDFLAGS="$saved_LDFLAGS -Wl,-rpath,$ledir/lib"])]) else ac_cv_libevent_dir="(system)" fi le_found=yes break fi done LIBS="$saved_LIBS" LDFLAGS="$saved_LDFLAGS" CPPFLAGS="$saved_CPPFLAGS" if test $le_found = no ; then AC_MSG_ERROR([libevent is required. You can get it from $LIBEVENT_URL If it's already installed, specify its path using --with-libevent=/dir/ ]) fi ]) LIBS="-levent $LIBS" if test $ac_cv_libevent_dir != "(system)"; then if test -d "$ac_cv_libevent_dir/lib" ; then LDFLAGS="-L$ac_cv_libevent_dir/lib $LDFLAGS" le_libdir="$ac_cv_libevent_dir/lib" else LDFLAGS="-L$ac_cv_libevent_dir $LDFLAGS" le_libdir="$ac_cv_libevent_dir" fi if test -d "$ac_cv_libevent_dir/include" ; then CPPFLAGS="-I$ac_cv_libevent_dir/include $CPPFLAGS" else CPPFLAGS="-I$ac_cv_libevent_dir $CPPFLAGS" fi fi trylibssldir="" AC_ARG_WITH(libssl, [ --with-libssl=PATH Specify path to libssl installation ], [ if test "x$withval" != "xno" ; then trylibssldir=$withval fi ] ) dnl ---------------------------------------------------------------------------- dnl libssl detection. swiped from libevent. modified for openssl detection. OPENSSL_URL=https://www.openssl.org/ if test "x$enable_tls" = "xyes"; then AC_CACHE_CHECK([for libssl directory], ac_cv_libssl_dir, [ saved_LIBS="$LIBS" saved_LDFLAGS="$LDFLAGS" saved_CPPFLAGS="$CPPFLAGS" le_found=no for ledir in $trylibssldir "" $prefix /usr/local ; do LDFLAGS="$saved_LDFLAGS" LIBS="-lssl -lcrypto $saved_LIBS" # Skip the directory if it isn't there. if test ! -z "$ledir" -a ! -d "$ledir" ; then continue; fi if test ! -z "$ledir" ; then if test -d "$ledir/lib" ; then LDFLAGS="-L$ledir/lib $LDFLAGS" else LDFLAGS="-L$ledir $LDFLAGS" fi if test -d "$ledir/include" ; then CPPFLAGS="-I$ledir/include $CPPFLAGS" else CPPFLAGS="-I$ledir $CPPFLAGS" fi fi # Can I compile and link it? AC_TRY_LINK([#include #include #include #include ], [ SSL_CTX* ssl_ctx = SSL_CTX_new(TLS_server_method()); assert(OPENSSL_VERSION_NUMBER >= 0x10100000L);], [ libssl_linked=yes ], [ libssl_linked=no ]) if test $libssl_linked = yes; then if test ! -z "$ledir" ; then ac_cv_libssl_dir=$ledir _myos=`echo $target_os | cut -f 1 -d .` AS_IF(test "$SUNCC" = "yes" -o "x$_myos" = "xsolaris2", [saved_LDFLAGS="$saved_LDFLAGS -Wl,-R$ledir/lib"], [AS_IF(test "$GCC" = "yes", [saved_LDFLAGS="$saved_LDFLAGS -Wl,-rpath,$ledir/lib"])]) else ac_cv_libssl_dir="(system)" fi le_found=yes break fi done LIBS="$saved_LIBS" LDFLAGS="$saved_LDFLAGS" CPPFLAGS="$saved_CPPFLAGS" if test $le_found = no ; then AC_MSG_ERROR([libssl (at least version 1.1.0) is required. You can get it from $OPENSSL_URL If it's already installed, specify its path using --with-libssl=/dir/ ]) fi ]) LIBS="-lssl -lcrypto $LIBS" if test $ac_cv_libssl_dir != "(system)"; then if test -d "$ac_cv_libssl_dir/lib" ; then LDFLAGS="-L$ac_cv_libssl_dir/lib $LDFLAGS" le_libdir="$ac_cv_libssl_dir/lib" else LDFLAGS="-L$ac_cv_libssl_dir $LDFLAGS" le_libdir="$ac_cv_libssl_dir" fi if test -d "$ac_cv_libssl_dir/include" ; then CPPFLAGS="-I$ac_cv_libssl_dir/include $CPPFLAGS" else CPPFLAGS="-I$ac_cv_libssl_dir $CPPFLAGS" fi fi fi dnl ---------------------------------------------------------------------------- AC_SEARCH_LIBS(umem_cache_create, umem) AC_SEARCH_LIBS(gethugepagesizes, hugetlbfs) AC_HEADER_STDBOOL AH_BOTTOM([#if HAVE_STDBOOL_H #include #else #define bool char #define false 0 #define true 1 #endif ]) AC_CHECK_HEADERS([inttypes.h]) AH_BOTTOM([#ifdef HAVE_INTTYPES_H #include #endif ]) dnl ********************************************************************** dnl Figure out if this system has the stupid sasl_callback_ft dnl ********************************************************************** AC_DEFUN([AC_HAVE_SASL_CALLBACK_FT], [AC_CACHE_CHECK(for sasl_callback_ft, ac_cv_has_sasl_callback_ft, [ AC_TRY_COMPILE([ #ifdef HAVE_SASL_SASL_H #include #include #endif ],[ sasl_callback_ft a_callback; ],[ ac_cv_has_sasl_callback_ft=yes ],[ ac_cv_has_sasl_callback_ft=no ]) ]) if test $ac_cv_has_sasl_callback_ft = yes; then AC_DEFINE(HAVE_SASL_CALLBACK_FT, 1, [we have sasl_callback_ft]) fi ]) AC_HAVE_SASL_CALLBACK_FT dnl ********************************************************************** dnl DETECT_UINT64_SUPPORT dnl dnl check if we can use a uint64_t dnl ********************************************************************** AC_DEFUN([AC_C_DETECT_UINT64_SUPPORT], [ AC_CACHE_CHECK([for print macros for integers (C99 section 7.8.1)], [ac_cv_c_uint64_support], [AC_TRY_COMPILE( [ #ifdef HAVE_INTTYPES_H #include #endif #include ], [ uint64_t val = 0; fprintf(stderr, "%" PRIu64 "\n", val); ], [ ac_cv_c_uint64_support=yes ], [ ac_cv_c_uint64_support=no ]) ]) ]) AC_C_DETECT_UINT64_SUPPORT AS_IF([test "x$ac_cv_c_uint64_support" = "xno"], [AC_MSG_WARN([ Failed to use print macros (PRIu) as defined in C99 section 7.8.1. ])]) AC_C_CONST dnl From licq: Copyright (c) 2000 Dirk Mueller dnl Check if the type socklen_t is defined anywhere AC_DEFUN([AC_C_SOCKLEN_T], [AC_CACHE_CHECK(for socklen_t, ac_cv_c_socklen_t, [ AC_TRY_COMPILE([ #include #include ],[ socklen_t foo; ],[ ac_cv_c_socklen_t=yes ],[ ac_cv_c_socklen_t=no ]) ]) if test $ac_cv_c_socklen_t = no; then AC_DEFINE(socklen_t, int, [define to int if socklen_t not available]) fi ]) AC_C_SOCKLEN_T dnl Check if we're a little-endian or a big-endian system, needed by hash code AC_C_BIGENDIAN( [AC_DEFINE(ENDIAN_BIG, 1, [machine is bigendian])], [AC_DEFINE(ENDIAN_LITTLE, 1, [machine is littleendian])], [AC_MSG_ERROR([Cannot detect endianness. Must pass ac_cv_c_bigendian={yes,no} to configure.])]) AC_DEFUN([AC_C_HTONLL], [ AC_MSG_CHECKING([for htonll]) have_htoll="no" AC_TRY_LINK([ #include #include #ifdef HAVE_INTTYPES_H #include */ #endif ], [ return htonll(0); ], [ have_htoll="yes" AC_DEFINE([HAVE_HTONLL], [1], [Have ntohll]) ], [ have_htoll="no" ]) AC_MSG_RESULT([$have_htoll]) ]) AC_C_HTONLL dnl Check whether the user's system supports pthread AC_SEARCH_LIBS(pthread_create, pthread) if test "x$ac_cv_search_pthread_create" = "xno"; then AC_MSG_ERROR([Can't enable threads without the POSIX thread library.]) fi AC_CHECK_FUNCS(mlockall) AC_CHECK_FUNCS(getpagesizes) AC_CHECK_FUNCS(sysconf) AC_CHECK_FUNCS(memcntl) AC_CHECK_FUNCS(sigignore) AC_CHECK_FUNCS(clock_gettime) AC_CHECK_FUNCS([accept4], [AC_DEFINE(HAVE_ACCEPT4, 1, [Define to 1 if support accept4])]) AC_CHECK_FUNCS([getopt_long], [AC_DEFINE(HAVE_GETOPT_LONG, 1, [Define to 1 if support getopt_long])]) dnl Need to disable opt for alignment check. GCC is too clever and turns this dnl into wide stores and no cmp under O2. AC_DEFUN([AC_C_ALIGNMENT], [AC_CACHE_CHECK(for alignment, ac_cv_c_alignment, [ AC_RUN_IFELSE( [AC_LANG_PROGRAM([ #include #include #pragma GCC optimize ("O0") ], [ char *buf = malloc(32); uint64_t *ptr = (uint64_t*)(buf+2); // catch sigbus, etc. *ptr = 0x1; // catch unaligned word access (ARM cpus) #ifdef ENDIAN_BIG #define ALIGNMENT 0x02030405 #else #define ALIGNMENT 0x05040302 #endif *(buf + 0) = 1; *(buf + 1) = 2; *(buf + 2) = 3; *(buf + 3) = 4; *(buf + 4) = 5; int* i = (int*)(buf+1); return (ALIGNMENT == *i) ? 0 : 1; ]) ],[ ac_cv_c_alignment=none ],[ ac_cv_c_alignment=need ],[ dnl cross compile ac_cv_c_alignment=maybe ]) ]) AS_IF([test $ac_cv_c_alignment = need], [AC_DEFINE(NEED_ALIGN, 1, [Machine need alignment])]) AS_IF([test $ac_cv_c_alignment = maybe], [AC_MSG_WARN([Assuming aligned access is required when cross-compiling]) AC_DEFINE(NEED_ALIGN, 1, [Machine need alignment])]) ]) AC_C_ALIGNMENT dnl Check for our specific usage of GCC atomics. dnl These were added in 4.1.2, but 32bit OS's may lack shorts and 4.1.2 dnl lacks testable defines. have_gcc_atomics=no AC_MSG_CHECKING(for GCC atomics) AC_TRY_LINK([],[ unsigned short a; unsigned short b; b = __sync_add_and_fetch(&a, 1); b = __sync_sub_and_fetch(&a, 2); ],[have_gcc_atomics=yes AC_DEFINE(HAVE_GCC_ATOMICS, 1, [GCC Atomics available])]) AC_MSG_RESULT($have_gcc_atomics) dnl Check for usage of 64bit atomics dnl 32bit systems shouldn't have these. have_gcc_64atomics=no AC_MSG_CHECKING(for GCC 64bit atomics) AC_TRY_LINK([#include ],[ uint64_t a; uint64_t b; b = __sync_add_and_fetch(&a, 1); b = __sync_sub_and_fetch(&a, 2); ],[have_gcc_64atomics=yes AC_DEFINE(HAVE_GCC_64ATOMICS, 1, [GCC 64bit Atomics available])]) AC_MSG_RESULT($have_gcc_64atomics) dnl Check for the requirements for running memcached with less privileges dnl than the default privilege set. On Solaris we need setppriv and priv.h dnl If you want to add support for other platforms you should check for dnl your requirements, define HAVE_DROP_PRIVILEGES, and make sure you add dnl the source file containing the implementation into memcached_SOURCE dnl in Makefile.am AC_CHECK_FUNCS(setppriv, [ AC_CHECK_HEADER(priv.h, [ AC_DEFINE([HAVE_DROP_PRIVILEGES], 1, [Define this if you have an implementation of drop_privileges()]) build_solaris_privs=yes ], []) ],[]) AS_IF([test "x$enable_seccomp" = "xyes" ], [ AC_CHECK_LIB(seccomp, seccomp_rule_add, [ AC_DEFINE([HAVE_DROP_PRIVILEGES], 1, [Define this if you have an implementation of drop_privileges()]) build_linux_privs=yes AC_DEFINE([HAVE_DROP_WORKER_PRIVILEGES], 1, [Define this if you have an implementation of drop_worker_privileges()]) build_linux_privs=yes ], []) ]) AC_CHECK_FUNCS(pledge, [ AC_CHECK_HEADER(unistd.h, [ AC_DEFINE([HAVE_DROP_PRIVILEGES], 1, [Define this if you have an implementation of drop_privileges()]) build_openbsd_privs=yes ], []) ],[]) AC_CHECK_FUNCS(cap_enter, [ AC_CHECK_HEADER(sys/capsicum.h, [ AC_DEFINE([HAVE_DROP_PRIVILEGES], 1, [Define this if you have an implementation of drop_privileges()]) build_freebsd_privs=yes ], []) ],[]) AM_CONDITIONAL([BUILD_SOLARIS_PRIVS],[test "$build_solaris_privs" = "yes"]) AM_CONDITIONAL([BUILD_LINUX_PRIVS],[test "$build_linux_privs" = "yes"]) AM_CONDITIONAL([BUILD_OPENBSD_PRIVS],[test "$build_openbsd_privs" = "yes"]) AM_CONDITIONAL([BUILD_FREEBSD_PRIVS],[test "$build_freebsd_privs" = "yes"]) AC_CHECK_HEADER(umem.h, [ AC_DEFINE([HAVE_UMEM_H], 1, [Define this if you have umem.h]) build_cache=no ], [build_cache=yes]) AM_CONDITIONAL([BUILD_CACHE], [test "x$build_cache" = "xyes"]) AC_ARG_ENABLE(docs, [AS_HELP_STRING([--disable-docs],[Disable documentation generation])]) AC_PATH_PROG([XML2RFC], [xml2rfc], "no") AC_PATH_PROG([XSLTPROC], [xsltproc], "no") AM_CONDITIONAL([BUILD_SPECIFICATIONS], [test "x$enable_docs" != "xno" -a "x$XML2RFC" != "xno" -a "x$XSLTPROC" != "xno"]) dnl Let the compiler be a bit more picky. Please note that you cannot dnl specify these flags to the compiler before AC_CHECK_FUNCS, because dnl the test program will generate a compilation warning and hence fail dnl to detect the function ;-) if test "$ICC" = "yes" then dnl ICC trying to be gcc. CFLAGS="$CFLAGS -diag-disable 187 -Wall -Werror" AC_DEFINE([_GNU_SOURCE],[1],[find sigignore on Linux]) elif test "$GCC" = "yes" then GCC_VERSION=`$CC -dumpversion` CFLAGS="$CFLAGS -Wall -Werror -pedantic -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls" if test "x$enable_asan" = "xyes"; then CFLAGS="$CFLAGS -fsanitize=address" fi case $GCC_VERSION in 4.4.*) CFLAGS="$CFLAGS -fno-strict-aliasing" ;; esac AC_DEFINE([_GNU_SOURCE],[1],[find sigignore on Linux]) elif test "$SUNCC" = "yes" then CFLAGS="$CFLAGS -errfmt=error -errwarn -errshort=tags" fi AC_CONFIG_FILES(Makefile doc/Makefile) AC_OUTPUT memcached-1.5.22/thread.c0000664000175000017500000006211013615376577012071 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ /* * Thread management for memcached. */ #include "memcached.h" #ifdef EXTSTORE #include "storage.h" #endif #include #include #include #include #include #include #ifdef __sun #include #endif #ifdef TLS #include #endif #define ITEMS_PER_ALLOC 64 /* An item in the connection queue. */ enum conn_queue_item_modes { queue_new_conn, /* brand new connection. */ queue_redispatch, /* redispatching from side thread */ }; typedef struct conn_queue_item CQ_ITEM; struct conn_queue_item { int sfd; enum conn_states init_state; int event_flags; int read_buffer_size; enum network_transport transport; enum conn_queue_item_modes mode; conn *c; void *ssl; CQ_ITEM *next; }; /* A connection queue. */ typedef struct conn_queue CQ; struct conn_queue { CQ_ITEM *head; CQ_ITEM *tail; pthread_mutex_t lock; }; /* Locks for cache LRU operations */ pthread_mutex_t lru_locks[POWER_LARGEST]; /* Connection lock around accepting new connections */ pthread_mutex_t conn_lock = PTHREAD_MUTEX_INITIALIZER; #if !defined(HAVE_GCC_ATOMICS) && !defined(__sun) pthread_mutex_t atomics_mutex = PTHREAD_MUTEX_INITIALIZER; #endif /* Lock for global stats */ static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; /* Lock to cause worker threads to hang up after being woken */ static pthread_mutex_t worker_hang_lock; /* Free list of CQ_ITEM structs */ static CQ_ITEM *cqi_freelist; static pthread_mutex_t cqi_freelist_lock; static pthread_mutex_t *item_locks; /* size of the item lock hash table */ static uint32_t item_lock_count; unsigned int item_lock_hashpower; #define hashsize(n) ((unsigned long int)1<<(n)) #define hashmask(n) (hashsize(n)-1) /* * Each libevent instance has a wakeup pipe, which other threads * can use to signal that they've put a new connection on its queue. */ static LIBEVENT_THREAD *threads; /* * Number of worker threads that have finished setting themselves up. */ static int init_count = 0; static pthread_mutex_t init_lock; static pthread_cond_t init_cond; static void thread_libevent_process(int fd, short which, void *arg); /* item_lock() must be held for an item before any modifications to either its * associated hash bucket, or the structure itself. * LRU modifications must hold the item lock, and the LRU lock. * LRU's accessing items must item_trylock() before modifying an item. * Items accessible from an LRU must not be freed or modified * without first locking and removing from the LRU. */ void item_lock(uint32_t hv) { mutex_lock(&item_locks[hv & hashmask(item_lock_hashpower)]); } void *item_trylock(uint32_t hv) { pthread_mutex_t *lock = &item_locks[hv & hashmask(item_lock_hashpower)]; if (pthread_mutex_trylock(lock) == 0) { return lock; } return NULL; } void item_trylock_unlock(void *lock) { mutex_unlock((pthread_mutex_t *) lock); } void item_unlock(uint32_t hv) { mutex_unlock(&item_locks[hv & hashmask(item_lock_hashpower)]); } static void wait_for_thread_registration(int nthreads) { while (init_count < nthreads) { pthread_cond_wait(&init_cond, &init_lock); } } static void register_thread_initialized(void) { pthread_mutex_lock(&init_lock); init_count++; pthread_cond_signal(&init_cond); pthread_mutex_unlock(&init_lock); /* Force worker threads to pile up if someone wants us to */ pthread_mutex_lock(&worker_hang_lock); pthread_mutex_unlock(&worker_hang_lock); } /* Must not be called with any deeper locks held */ void pause_threads(enum pause_thread_types type) { char buf[1]; int i; buf[0] = 0; switch (type) { case PAUSE_ALL_THREADS: slabs_rebalancer_pause(); lru_maintainer_pause(); lru_crawler_pause(); #ifdef EXTSTORE storage_compact_pause(); storage_write_pause(); #endif case PAUSE_WORKER_THREADS: buf[0] = 'p'; pthread_mutex_lock(&worker_hang_lock); break; case RESUME_ALL_THREADS: slabs_rebalancer_resume(); lru_maintainer_resume(); lru_crawler_resume(); #ifdef EXTSTORE storage_compact_resume(); storage_write_resume(); #endif case RESUME_WORKER_THREADS: pthread_mutex_unlock(&worker_hang_lock); break; default: fprintf(stderr, "Unknown lock type: %d\n", type); assert(1 == 0); break; } /* Only send a message if we have one. */ if (buf[0] == 0) { return; } pthread_mutex_lock(&init_lock); init_count = 0; for (i = 0; i < settings.num_threads; i++) { if (write(threads[i].notify_send_fd, buf, 1) != 1) { perror("Failed writing to notify pipe"); /* TODO: This is a fatal problem. Can it ever happen temporarily? */ } } wait_for_thread_registration(settings.num_threads); pthread_mutex_unlock(&init_lock); } // MUST not be called with any deeper locks held // MUST be called only by parent thread // Note: listener thread is the "main" event base, which has exited its // loop in order to call this function. void stop_threads(void) { char buf[1]; int i; // assoc can call pause_threads(), so we have to stop it first. stop_assoc_maintenance_thread(); if (settings.verbose > 0) fprintf(stderr, "stopped assoc\n"); if (settings.verbose > 0) fprintf(stderr, "asking workers to stop\n"); buf[0] = 's'; pthread_mutex_lock(&init_lock); init_count = 0; for (i = 0; i < settings.num_threads; i++) { if (write(threads[i].notify_send_fd, buf, 1) != 1) { perror("Failed writing to notify pipe"); /* TODO: This is a fatal problem. Can it ever happen temporarily? */ } } wait_for_thread_registration(settings.num_threads); pthread_mutex_unlock(&init_lock); if (settings.verbose > 0) fprintf(stderr, "asking background threads to stop\n"); // stop each side thread. // TODO: Verify these all work if the threads are already stopped stop_item_crawler_thread(CRAWLER_WAIT); if (settings.verbose > 0) fprintf(stderr, "stopped lru crawler\n"); stop_lru_maintainer_thread(); if (settings.verbose > 0) fprintf(stderr, "stopped maintainer\n"); stop_slab_maintenance_thread(); if (settings.verbose > 0) fprintf(stderr, "stopped slab mover\n"); logger_stop(); if (settings.verbose > 0) fprintf(stderr, "stopped logger thread\n"); stop_conn_timeout_thread(); if (settings.verbose > 0) fprintf(stderr, "stopped idle timeout thread\n"); if (settings.verbose > 0) fprintf(stderr, "all background threads stopped\n"); // At this point, every background thread must be stopped. } /* * Initializes a connection queue. */ static void cq_init(CQ *cq) { pthread_mutex_init(&cq->lock, NULL); cq->head = NULL; cq->tail = NULL; } /* * Looks for an item on a connection queue, but doesn't block if there isn't * one. * Returns the item, or NULL if no item is available */ static CQ_ITEM *cq_pop(CQ *cq) { CQ_ITEM *item; pthread_mutex_lock(&cq->lock); item = cq->head; if (NULL != item) { cq->head = item->next; if (NULL == cq->head) cq->tail = NULL; } pthread_mutex_unlock(&cq->lock); return item; } /* * Adds an item to a connection queue. */ static void cq_push(CQ *cq, CQ_ITEM *item) { item->next = NULL; pthread_mutex_lock(&cq->lock); if (NULL == cq->tail) cq->head = item; else cq->tail->next = item; cq->tail = item; pthread_mutex_unlock(&cq->lock); } /* * Returns a fresh connection queue item. */ static CQ_ITEM *cqi_new(void) { CQ_ITEM *item = NULL; pthread_mutex_lock(&cqi_freelist_lock); if (cqi_freelist) { item = cqi_freelist; cqi_freelist = item->next; } pthread_mutex_unlock(&cqi_freelist_lock); if (NULL == item) { int i; /* Allocate a bunch of items at once to reduce fragmentation */ item = malloc(sizeof(CQ_ITEM) * ITEMS_PER_ALLOC); if (NULL == item) { STATS_LOCK(); stats.malloc_fails++; STATS_UNLOCK(); return NULL; } /* * Link together all the new items except the first one * (which we'll return to the caller) for placement on * the freelist. */ for (i = 2; i < ITEMS_PER_ALLOC; i++) item[i - 1].next = &item[i]; pthread_mutex_lock(&cqi_freelist_lock); item[ITEMS_PER_ALLOC - 1].next = cqi_freelist; cqi_freelist = &item[1]; pthread_mutex_unlock(&cqi_freelist_lock); } return item; } /* * Frees a connection queue item (adds it to the freelist.) */ static void cqi_free(CQ_ITEM *item) { pthread_mutex_lock(&cqi_freelist_lock); item->next = cqi_freelist; cqi_freelist = item; pthread_mutex_unlock(&cqi_freelist_lock); } /* * Creates a worker thread. */ static void create_worker(void *(*func)(void *), void *arg) { pthread_attr_t attr; int ret; pthread_attr_init(&attr); if ((ret = pthread_create(&((LIBEVENT_THREAD*)arg)->thread_id, &attr, func, arg)) != 0) { fprintf(stderr, "Can't create thread: %s\n", strerror(ret)); exit(1); } } /* * Sets whether or not we accept new connections. */ void accept_new_conns(const bool do_accept) { pthread_mutex_lock(&conn_lock); do_accept_new_conns(do_accept); pthread_mutex_unlock(&conn_lock); } /****************************** LIBEVENT THREADS *****************************/ /* * Set up a thread's information. */ static void setup_thread(LIBEVENT_THREAD *me) { #if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= 0x02000101 struct event_config *ev_config; ev_config = event_config_new(); event_config_set_flag(ev_config, EVENT_BASE_FLAG_NOLOCK); me->base = event_base_new_with_config(ev_config); event_config_free(ev_config); #else me->base = event_init(); #endif if (! me->base) { fprintf(stderr, "Can't allocate event base\n"); exit(1); } /* Listen for notifications from other threads */ event_set(&me->notify_event, me->notify_receive_fd, EV_READ | EV_PERSIST, thread_libevent_process, me); event_base_set(me->base, &me->notify_event); if (event_add(&me->notify_event, 0) == -1) { fprintf(stderr, "Can't monitor libevent notify pipe\n"); exit(1); } me->new_conn_queue = malloc(sizeof(struct conn_queue)); if (me->new_conn_queue == NULL) { perror("Failed to allocate memory for connection queue"); exit(EXIT_FAILURE); } cq_init(me->new_conn_queue); if (pthread_mutex_init(&me->stats.mutex, NULL) != 0) { perror("Failed to initialize mutex"); exit(EXIT_FAILURE); } me->suffix_cache = cache_create("suffix", SUFFIX_SIZE, sizeof(char*), NULL, NULL); if (me->suffix_cache == NULL) { fprintf(stderr, "Failed to create suffix cache\n"); exit(EXIT_FAILURE); } #ifdef EXTSTORE me->io_cache = cache_create("io", sizeof(io_wrap), sizeof(char*), NULL, NULL); if (me->io_cache == NULL) { fprintf(stderr, "Failed to create IO object cache\n"); exit(EXIT_FAILURE); } #endif #ifdef TLS if (settings.ssl_enabled) { me->ssl_wbuf = (char *)malloc((size_t)settings.ssl_wbuf_size); if (me->ssl_wbuf == NULL) { fprintf(stderr, "Failed to allocate the SSL write buffer\n"); exit(EXIT_FAILURE); } } #endif } /* * Worker thread: main event loop */ static void *worker_libevent(void *arg) { LIBEVENT_THREAD *me = arg; /* Any per-thread setup can happen here; memcached_thread_init() will block until * all threads have finished initializing. */ me->l = logger_create(); me->lru_bump_buf = item_lru_bump_buf_create(); if (me->l == NULL || me->lru_bump_buf == NULL) { abort(); } if (settings.drop_privileges) { drop_worker_privileges(); } register_thread_initialized(); event_base_loop(me->base, 0); // same mechanism used to watch for all threads exiting. register_thread_initialized(); event_base_free(me->base); return NULL; } /* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *item; char buf[1]; conn *c; unsigned int timeout_fd; if (read(fd, buf, 1) != 1) { if (settings.verbose > 0) fprintf(stderr, "Can't read from libevent pipe\n"); return; } switch (buf[0]) { case 'c': item = cq_pop(me->new_conn_queue); if (NULL == item) { break; } switch (item->mode) { case queue_new_conn: c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base, item->ssl); if (c == NULL) { if (IS_UDP(item->transport)) { fprintf(stderr, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { fprintf(stderr, "Can't listen for events on fd %d\n", item->sfd); } #ifdef TLS if (item->ssl) { SSL_shutdown(item->ssl); SSL_free(item->ssl); } #endif close(item->sfd); } } else { c->thread = me; #ifdef TLS if (settings.ssl_enabled && c->ssl != NULL) { assert(c->thread && c->thread->ssl_wbuf); c->ssl_wbuf = c->thread->ssl_wbuf; } #endif } break; case queue_redispatch: conn_worker_readd(item->c); break; } cqi_free(item); break; /* we were told to pause and report in */ case 'p': register_thread_initialized(); break; /* a client socket timed out */ case 't': if (read(fd, &timeout_fd, sizeof(timeout_fd)) != sizeof(timeout_fd)) { if (settings.verbose > 0) fprintf(stderr, "Can't read timeout fd from libevent pipe\n"); return; } conn_close_idle(conns[timeout_fd]); break; /* asked to stop */ case 's': event_base_loopexit(me->base, NULL); break; } } /* Which thread we assigned a connection to most recently. */ static int last_thread = -1; /* * Dispatches a new connection to another thread. This is only ever called * from the main thread, either during initialization (for UDP) or because * of an incoming connection. */ void dispatch_conn_new(int sfd, enum conn_states init_state, int event_flags, int read_buffer_size, enum network_transport transport, void *ssl) { CQ_ITEM *item = cqi_new(); char buf[1]; if (item == NULL) { close(sfd); /* given that malloc failed this may also fail, but let's try */ fprintf(stderr, "Failed to allocate memory for connection object\n"); return; } int tid = (last_thread + 1) % settings.num_threads; LIBEVENT_THREAD *thread = threads + tid; last_thread = tid; item->sfd = sfd; item->init_state = init_state; item->event_flags = event_flags; item->read_buffer_size = read_buffer_size; item->transport = transport; item->mode = queue_new_conn; item->ssl = ssl; cq_push(thread->new_conn_queue, item); MEMCACHED_CONN_DISPATCH(sfd, (int64_t)thread->thread_id); buf[0] = 'c'; if (write(thread->notify_send_fd, buf, 1) != 1) { perror("Writing to thread notify pipe"); } } /* * Re-dispatches a connection back to the original thread. Can be called from * any side thread borrowing a connection. */ void redispatch_conn(conn *c) { CQ_ITEM *item = cqi_new(); char buf[1]; if (item == NULL) { /* Can't cleanly redispatch connection. close it forcefully. */ c->state = conn_closed; close(c->sfd); return; } LIBEVENT_THREAD *thread = c->thread; item->sfd = c->sfd; item->init_state = conn_new_cmd; item->c = c; item->mode = queue_redispatch; cq_push(thread->new_conn_queue, item); buf[0] = 'c'; if (write(thread->notify_send_fd, buf, 1) != 1) { perror("Writing to thread notify pipe"); } } /* This misses the allow_new_conns flag :( */ void sidethread_conn_close(conn *c) { c->state = conn_closed; if (settings.verbose > 1) fprintf(stderr, "<%d connection closed from side thread.\n", c->sfd); #ifdef TLS if (c->ssl) { c->ssl_wbuf = NULL; SSL_shutdown(c->ssl); SSL_free(c->ssl); } #endif close(c->sfd); STATS_LOCK(); stats_state.curr_conns--; STATS_UNLOCK(); return; } /********************************* ITEM ACCESS *******************************/ /* * Allocates a new item. */ item *item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes) { item *it; /* do_item_alloc handles its own locks */ it = do_item_alloc(key, nkey, flags, exptime, nbytes); return it; } /* * Returns an item if it hasn't been marked as expired, * lazy-expiring as needed. */ item *item_get(const char *key, const size_t nkey, conn *c, const bool do_update) { item *it; uint32_t hv; hv = hash(key, nkey); item_lock(hv); it = do_item_get(key, nkey, hv, c, do_update); item_unlock(hv); return it; } // returns an item with the item lock held. // lock will still be held even if return is NULL, allowing caller to replace // an item atomically if desired. item *item_get_locked(const char *key, const size_t nkey, conn *c, const bool do_update, uint32_t *hv) { item *it; *hv = hash(key, nkey); item_lock(*hv); it = do_item_get(key, nkey, *hv, c, do_update); return it; } item *item_touch(const char *key, size_t nkey, uint32_t exptime, conn *c) { item *it; uint32_t hv; hv = hash(key, nkey); item_lock(hv); it = do_item_touch(key, nkey, exptime, hv, c); item_unlock(hv); return it; } /* * Links an item into the LRU and hashtable. */ int item_link(item *item) { int ret; uint32_t hv; hv = hash(ITEM_key(item), item->nkey); item_lock(hv); ret = do_item_link(item, hv); item_unlock(hv); return ret; } /* * Decrements the reference count on an item and adds it to the freelist if * needed. */ void item_remove(item *item) { uint32_t hv; hv = hash(ITEM_key(item), item->nkey); item_lock(hv); do_item_remove(item); item_unlock(hv); } /* * Replaces one item with another in the hashtable. * Unprotected by a mutex lock since the core server does not require * it to be thread-safe. */ int item_replace(item *old_it, item *new_it, const uint32_t hv) { return do_item_replace(old_it, new_it, hv); } /* * Unlinks an item from the LRU and hashtable. */ void item_unlink(item *item) { uint32_t hv; hv = hash(ITEM_key(item), item->nkey); item_lock(hv); do_item_unlink(item, hv); item_unlock(hv); } /* * Does arithmetic on a numeric item value. */ enum delta_result_type add_delta(conn *c, const char *key, const size_t nkey, bool incr, const int64_t delta, char *buf, uint64_t *cas) { enum delta_result_type ret; uint32_t hv; hv = hash(key, nkey); item_lock(hv); ret = do_add_delta(c, key, nkey, incr, delta, buf, cas, hv); item_unlock(hv); return ret; } /* * Stores an item in the cache (high level, obeys set/add/replace semantics) */ enum store_item_type store_item(item *item, int comm, conn* c) { enum store_item_type ret; uint32_t hv; hv = hash(ITEM_key(item), item->nkey); item_lock(hv); ret = do_store_item(item, comm, c, hv); item_unlock(hv); return ret; } /******************************* GLOBAL STATS ******************************/ void STATS_LOCK() { pthread_mutex_lock(&stats_lock); } void STATS_UNLOCK() { pthread_mutex_unlock(&stats_lock); } void threadlocal_stats_reset(void) { int ii; for (ii = 0; ii < settings.num_threads; ++ii) { pthread_mutex_lock(&threads[ii].stats.mutex); #define X(name) threads[ii].stats.name = 0; THREAD_STATS_FIELDS #ifdef EXTSTORE EXTSTORE_THREAD_STATS_FIELDS #endif #undef X memset(&threads[ii].stats.slab_stats, 0, sizeof(threads[ii].stats.slab_stats)); memset(&threads[ii].stats.lru_hits, 0, sizeof(uint64_t) * POWER_LARGEST); pthread_mutex_unlock(&threads[ii].stats.mutex); } } void threadlocal_stats_aggregate(struct thread_stats *stats) { int ii, sid; /* The struct has a mutex, but we can safely set the whole thing * to zero since it is unused when aggregating. */ memset(stats, 0, sizeof(*stats)); for (ii = 0; ii < settings.num_threads; ++ii) { pthread_mutex_lock(&threads[ii].stats.mutex); #define X(name) stats->name += threads[ii].stats.name; THREAD_STATS_FIELDS #ifdef EXTSTORE EXTSTORE_THREAD_STATS_FIELDS #endif #undef X for (sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) { #define X(name) stats->slab_stats[sid].name += \ threads[ii].stats.slab_stats[sid].name; SLAB_STATS_FIELDS #undef X } for (sid = 0; sid < POWER_LARGEST; sid++) { stats->lru_hits[sid] += threads[ii].stats.lru_hits[sid]; stats->slab_stats[CLEAR_LRU(sid)].get_hits += threads[ii].stats.lru_hits[sid]; } pthread_mutex_unlock(&threads[ii].stats.mutex); } } void slab_stats_aggregate(struct thread_stats *stats, struct slab_stats *out) { int sid; memset(out, 0, sizeof(*out)); for (sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) { #define X(name) out->name += stats->slab_stats[sid].name; SLAB_STATS_FIELDS #undef X } } /* * Initializes the thread subsystem, creating various worker threads. * * nthreads Number of worker event handler threads to spawn */ void memcached_thread_init(int nthreads, void *arg) { int i; int power; for (i = 0; i < POWER_LARGEST; i++) { pthread_mutex_init(&lru_locks[i], NULL); } pthread_mutex_init(&worker_hang_lock, NULL); pthread_mutex_init(&init_lock, NULL); pthread_cond_init(&init_cond, NULL); pthread_mutex_init(&cqi_freelist_lock, NULL); cqi_freelist = NULL; /* Want a wide lock table, but don't waste memory */ if (nthreads < 3) { power = 10; } else if (nthreads < 4) { power = 11; } else if (nthreads < 5) { power = 12; } else if (nthreads <= 10) { power = 13; } else if (nthreads <= 20) { power = 14; } else { /* 32k buckets. just under the hashpower default. */ power = 15; } if (power >= hashpower) { fprintf(stderr, "Hash table power size (%d) cannot be equal to or less than item lock table (%d)\n", hashpower, power); fprintf(stderr, "Item lock table grows with `-t N` (worker threadcount)\n"); fprintf(stderr, "Hash table grows with `-o hashpower=N` \n"); exit(1); } item_lock_count = hashsize(power); item_lock_hashpower = power; item_locks = calloc(item_lock_count, sizeof(pthread_mutex_t)); if (! item_locks) { perror("Can't allocate item locks"); exit(1); } for (i = 0; i < item_lock_count; i++) { pthread_mutex_init(&item_locks[i], NULL); } threads = calloc(nthreads, sizeof(LIBEVENT_THREAD)); if (! threads) { perror("Can't allocate thread descriptors"); exit(1); } for (i = 0; i < nthreads; i++) { int fds[2]; if (pipe(fds)) { perror("Can't create notify pipe"); exit(1); } threads[i].notify_receive_fd = fds[0]; threads[i].notify_send_fd = fds[1]; #ifdef EXTSTORE threads[i].storage = arg; #endif setup_thread(&threads[i]); /* Reserve three fds for the libevent base, and two for the pipe */ stats_state.reserved_fds += 5; } /* Create threads after we've done all the libevent setup. */ for (i = 0; i < nthreads; i++) { create_worker(worker_libevent, &threads[i]); } /* Wait for all the threads to set themselves up before returning. */ pthread_mutex_lock(&init_lock); wait_for_thread_registration(nthreads); pthread_mutex_unlock(&init_lock); } memcached-1.5.22/ChangeLog0000664000175000017500000004632513575022500012215 000000000000002016-08-23 * ChangeLog moved from Google Code to Github Wiki, it's now available at: https://github.com/memcached/memcached/wiki/ReleaseNotes 2010-10-11 * ChangeLog is no longer being updated. See http://code.google.com/p/memcached/wiki/ReleaseNotes 2009-04-10 * *critical bugfix*. In 1.2.7 under multithreaded mode, memcached would never restart accepting connections after hitting the maximum connection limit. 2009-04-02 * Fix: Rewrote stat buffer handling (Trond) 2009-03-31 * Feature: -b option for configuring backlog * Feature: stats settings * Fix: binary stat alignment bug (bug26) * Fix: buffer overflow in stats (bug27) * Fix: recycle memory more aggressively (bug14) * Fix: incr validation (bug31) * Fix: 64-bit incr/decr delta problem (bug21) * Fix: Ascii UDP set (bug36) * Fix: stats slabs' used chunks (bug29) * Fix: stats reset resetting more (bug22) * Misc: More tests, documentation, cleanliness (godliness) * Stable merge (stats, debuggability, listen bugs) 2009-03-11 * Protocol: Binary complete (Dustin, Trond, Toru, etc...) * Performance: improvements from facebook (Dormando) * Performance: per-thread stats (Trond) * Performance: Hash expansion in its own thread (Trond) * Stats: Lots of new stats (Dustin) * Bug fixes (various) See the 1.3.2 release notes or revision control for more detail. 2008-09-06 * Display error status on listen failures (Dormando) * Remove managed instance code. Incomplete/etc. (Dormando) * Handle broken IPV6 stacks better (Brian Aker) * Generate warnings on setsockopt() failures (Brian Aker) * Fix some indentation issues (Brian Aker) * UDP/TCP can be disabled by setting their port to zero (Brian Aker) * Zero out libevent thread structures before use (Ricky Zhou) * New stat: Last accessed time for last evicted item per slab class. (Dormando) * Use a dedicated socket accept thread (Facebook) * Add -R option. Limit the number of requests processed by a connection at once. Prevents starving other threads if bulk loading. (Facebook) 2008-07-29 [Version 1.2.6 released] 2008-07-24 [Version 1.2.6-rc1 released] * Add support for newer automake (Facebook) * DTrace support for Solaris/etc (Trond Norbye) * LRU tests (Steve Yen) * Handle negative length items properly (Dormando) * Don't leave stale data after failed set attempts (Dormando) * Fix refcount leaks, which would result in OOM's on all sets (Dormando) * Fix buffer overruns (Dustin Sallings, Tomash Brechko) * Fix memory corruption with CAS (Dustin Sallings) 2008-06-11 * Fix -k to work with -d. (reported by Gary Zhu) 2008-03-02 [Version 1.2.5-rc1 released] * Add per-item-class tracking of evictions and OOM errors (dormando) * Optimize item_alloc() a little (dormando) * Give 'SERVER_ERROR out of memory' errors more context (dormando) * Enable usage of large memory pages under solaris (Trond.Norbye@Sun.COM) * Enable UDP by default, clean up server socket code (brian@tangent.org) * 'noreply' support (Tomash Brechko) * IPv6 support, and IPv6 multi-interface support (brian@tangent.org) * Add compiler options for Sun Studio compilers with --enable-threads (Trond.Norbye@Sun.COM) * Add --enable-64bit for mulitarget platforms (Trond.Norbye@Sun.COM) * Use gettimeofday(2) instead of time(2). * Make -k option work (Tomash Brechko) * Fix chunk slab alignment (Trond.Norbye@Sun.COM) 2007-12-06 [Version 1.2.4 released] 2007-12-05 * Fix compilation on panther (JS and Dormando) * More CAS tests (Chris Goffinet) * Final fixes for all 1.2.4 features are in, -rc2 sent out. 2007-11-19 [Version 1.2.4-rc1 released] 2007-11-19 Dormando * Patch series from Tomash Brechko : Minor fixes and optimizations. * Patches from Chris, Dustin, and Dormando to fix CAS. * Prepping for 1.2.4 release. 2007-11-13 Dormando * Adjusted patch from js : Compile on OS X Panther and earlier. 2007-11-12 Steven Grimm * Patch from Tomash Brechko : Always send "SERVER_ERROR out of memory" when memory exhausted. 2007-10-15 Paul Lindner * Patch from David Bremner that implements a new option "-a" which takes an octal permission mask (like chmod) sets the permissions on the unix domain socket (specified by "-s"). 2007-10-03 Paul Lindner * Incorporate "cas" operation developed by Dustin Sallings This change allows you to do atomic changes to an existing key. * Fix for stats.evictions not incrementing when exptime == 0 items are kicked off the cache. from Jean-Francois BUSTARRET . * Fix for do_item_cachedump() which was returning an incorrect timestamp. * Switch to unsigned 64-bit increment/decrement counters from Evan Miller and Dustin Sallings. * Add append command support written by Filipe Laborde. Thread safe version plus prepend command from Maxim Dounin * The memcached-tool script can now display stats. Patch provided by Dan Christian * Fix for Unix Domain sockets on FreeBSD FreeBSD's sendmsg() requires msg_name in msghdr structure to be NULL if not used, setting msg_namelen to 0 isn't enough. Patch from Maxim Dounin 2007-08-21 Paul Lindner * Incorporate increment patch from Evan Miller to define increment overflow behavior. 2007-08-07 Leon Brocard * Bring the memcached.1 manpage up to date 2007-08-06 Paul Lindner * Fix crash when using -P and -d flags on x86_64 with latest libevent release. 2007-07-08 Steven Grimm * Item stats commands weren't thread-safe; wrap them with locks when compiled in multithreaded mode. * The "stats items" command now works again; it broke with the introduction of the powers-of-N chunk size change. 2007-07-06 [Version 1.2.3 released] 2007-06-19 Paul Lindner * Solaris portability fixes from Trond Norbye 2007-05-29 Paul Lindner * Properly document evictions statistic value 2007-05-10 Paul Lindner * Flesh out tests for unix domain sockets and binary data. * Update rpm spec file to run tests 2007-05-07 Paul Lindner * Fix compilation bug on freebsd 6.x (and maybe others) * Update RPM spec file per redhat bugzilla #238994 * Move unistd.h to memcached.h to get rid of warnings * Add string.h to thread.c to get correctly prototyped strerror() 2007-05-04 Paul Lindner * Add fedora/redhat style init script and RPM spec file 2007-05-12 [Version 1.2.2 released] 2007-04-16 Steven Grimm * Command tokenizer performance and cleanliness improvement. Patch contributed by Paolo Borelli . 2007-04-16 Paul Lindner * Add notes to README about MacOS, libevent and kqueue. * Windows Patch integration -- part 1, warnings elimination. 2007-04-12 Paul Lindner * Allow changes to the verbosity level of the server with a new "verbosity" command and some compiler cleanups. Patch contributed by Paolo Borelli . 2007-04-08 Paul Lindner * Add cleanup patch from "Tim Yardley" to clean up source spacing issues, fix -Wall warnings, add some null checks, adds asserts at the top of each function for any use of conn *c without checking to see if c is NULL first. * Also adjust clean-whitespace.pl to clean *.ac files. Add script to test-suite to test for tabs. 2007-04-04 Paul Lindner * Add clarification of flush_all in the protocol docs from Elizabeth Mattijsen 2007-03-31 Paul Lindner * Add patch from Eli Bingham to re-enable the -n switch to memcached. 2007-03-20 Paul Lindner * Add patch to collect eviction statistics from Jean-Francois BUSTARRET . * Updated docs, added new test cases for t/stats.t 2007-03-18 Paul Lindner * Add more test cases using larger buffer sizes up to and greater than 1MB. * Remove unused parameter to item_size_ok() * Use a single printf() in usage() * Add a failing test for conforming with maximum connections. 2007-03-17 * crash fix from Thomas van Gulick in conn_shrink(), passing &ptr, instead of ptr to realloc(). 2007-03-05 Paul Lindner * Fix a number of places where (s)printf calls were using unsigned or signed formats that did not match their arguments. * Add support for stdbool.h and stdint.h to use the bool and uint8_t types. * Major refactoring - move API calls for assoc/items/slabs to their own individual header files. Add appropriate const and static declarations as appropriate. * Avoid type-punning. Do a more efficient realloc inside the conn_shrink routine. * Fix overflow bug where uninitialized access to slabclass caused size-0 mallocs during slab preallocation. * Use EXIT_SUCCESS/EXIT_FAILURE constants. * Convert some sprintf calls to snprintf to protect against buffer overflows. * Explicitly compare against NULL or zero in many places. 2007-03-05 * Steven Grimm : Per-object-type stats collection support. Specify the object type delimiter with the -D command line option. Turn stats gathering on and off with "stats detail on" and "stats detail off". Dump the per-object-type details with "stats detail dump". 2007-03-01 * Steven Grimm : Fix an off-by-one error in the multithreaded version's message passing code. 2006-12-23 * fix expirations of items set with absolute expiration times in the past, before the server's start time. bug was introduced in 1.2.0 with rel_time_t. Thanks to Adam Dixon for the bug report and test case! 2006-11-26 * Steven Grimm : Performance improvements: Dynamic sizing of hashtable to reduce collisions on very large caches and conserve memory on small caches. Only reposition items in the LRU queue once a minute, to reduce overhead of accessing extremely frequently-used items. Stop listening for new connections until an existing one closes if we run out of available file descriptors. Command parser refactoring: Add a single-pass tokenizer to cut down on string scanning. Split the command processing into separate functions for easier profiling and better readability. Pass key lengths along with the keys in all API functions that need keys, to avoid needing to call strlen() repeatedly. 2006-11-25 * Steve Peters : OpenBSD has a malloc.h, but warns to use stdlib.h instead 2006-11-22 * Steven Grimm : Add support for multithreaded execution. Run configure with "--enable-threads" to enable. See doc/threads.txt for details. 2006-11-13 * Iain Wade : Fix for UDP responses on non-"get" commands. 2006-10-15 * Steven Grimm : Dynamic sizing of hashtable to reduce collisions on very large caches and conserve memory on small caches. 2006-10-13 * Steven Grimm : New faster hash function. 2006-09-20 * don't listen on UDP by default; more clear message when UDP port in use 2006-09-09 * release 1.2.0 (along with 1.1.13, which is the more tested branch) nobody has run 1.2.0 in production, to my knowledge. facebook has run their pre-merge-with-trunk version, but bugs were discovered (and fixed) after the merge. there might be more. you've been warned. :) 2006-09-04 * improved autoconf libevent detection, from the Tor project. 2006-09-03 * test suite and lot of expiration, delete, flush_all, etc corner case bugs fixed (Brad Fitzpatrick) 2006-09-02 * Nathan Neulinger : fix breakage in expiration code causing expiration times to not be processed correctly. 2006-08-21 * Nathan Neulinger : fix incompatibilities with unix domain socket support and the UDP code and clean up stale sockets 2006-08-20 * Nathan Neulinger : unix domain socket support 2006-05-03 * Steven Grimm : big bunch of changes: big CPU reduction work, UDP-based interface, increased memory efficiency. (intertwined patch, committed all together) or see svn commit logs 2006-04-30 * River Tarnell: autoconf work for Solaris 10. Brad: merge and verify it works on Nexenta. 2006-03-04 * avva: bucket/generation patch (old, but Brad's just finally committing it) 2006-01-01 * Brad Fitzpatrick : allocate 1 slab per class on start-up, to avoid confusing users with out-of-memory errors later. this is 18 MB of allocation on start, unless max memory allowed with -m is lower, in which case only the smaller slab classes are allocated. 2005-08-09 * Elizabeth Mattijsen : needed a way to flush all memcached backend servers, but not at exactly the same time (to reduce load peaks), I've added some simple functionality to the memcached protocol in the "flush_all" command that allows you to specify a time at which the flush will actually occur (instead of always at the moment the "flush_all" command is received). 2005-05-25 * patch from Peter van Dijk to make stderr unbuffered, for running under daemontools 2005-04-04 * patch from Don MacAskill 'flush_all' doesn't seem to work properly. Basically, if you try to add a key which is present, but expired, the store fails but the old key is no longer expired. * release 1.1.12 2005-01-14 * Date: Thu, 18 Nov 2004 15:25:59 -0600 From: David Phillips Here is a patch to configure.ac and Makefile.am to put the man page in the correct location. Trying to install the man page from a subdirectory results in the subdirectory being used in the install path (it tries to install to doc/memcached.1). This is the correct thing to do: - create a Makefile.am in the doc directory that installs the man page with man_MANS - modify Makefile.am in the base directory to reference the doc directory using SUBDIRS - modify the AC_CONFIG_FILES macro in configure.ac to output the Makefile in doc 2005-01-14 * pidfile saving support from Lisa Seelye , sent Jan 13, 2005 2005-01-14 * don't delete libevent events that haven't been added (the deltimer) patch from Ted Schundler 2004-12-10 * document -M and -r in manpage (Doug Porter ) 2004-07-22 * fix buffer overflow in items.c with 250 byte keys along with other info on the same line going into a 256 byte char[]. thanks to Andrei Nigmatulin 2004-06-15 * immediate deletes weren't being unlinked a few seconds, preventing "add" commands to the same key in that time period. thanks to Michael Alan Dorman for the bug report and demo script. 2004-04-30 * released 1.1.11 2004-04-24 * Avva: Add a new command line option: -r , to maximize core file limit. 2004-03-31 * Avva: Use getrlimit and setrlimit to set limits for number of simultaneously open file descriptors. Get the current limits and try to raise them if they're not enough for the specified (or the default) setting of max connections. 2004-02-24 * Adds a '-M' flag to turn off tossing items from the cache. (Jason Titus ) 2004-02-19 (Evan) * Install manpage on "make install", etc. 2003-12-30 (Brad) * remove static build stuff. interferes with PAM setuid stuff and was only included as a possible fix with the old memory allocator. really shouldn't make a difference. * add Jay Bonci's Debian scripts and manpage * release version 1.1.10 2003-12-01 (Avva) * New command: flush_all, causes all existing items to be invalidated immediately (without deleting them from memory, merely causing memcached to no longer return them). 2003-10-23 * Shift init code around to fix daemon mode on FreeBSD, * and drop root only after creating the server socket (to * allow the use of privileged ports) * version 1.1.10pre 2003-10-09 * BSD compile fixes from Ryan T. Dean * version 1.1.9 2003-09-29 * ignore SIGPIPE at start instead of crashing in rare cases it comes up. no other code had to be modified, since everything else is already dead-connection-aware. (avva) 2003-09-09 (Avva, Lisa Marie Seelye ) * setuid support 2003-09-05 (Avva) * accept all new connections in the same event (so we work with ET epoll) * mark all items as clsid=0 after slab page reassignment to please future asserts (on the road to making slab page reassignment work fully) 2003-08-12 (Brad Fitzpatrick) * use TCP_CORK on Linux or TCP_PUSH on BSD * only use TCP_NODELAY when we don't have alternatives 2003-08-10 * disable Nagel's Algorithm (TCP_NODELAY) for better performance (avva) 2003-08-10 * support multiple levels of verbosity (-vv) 2003-08-10 (Evan Martin) * Makefile.am: debug, optimization, and static flags are controlled by the configure script. * configure.ac: - allow specifying libevent directory with --with-libevent=DIR - check for malloc.h (unavailable on BSDs) - check for socklen_t (unavailable on OSX) * assoc.c, items.c, slabs.c: Remove some unused headers. * memcached.c: allow for nonexistence of malloc.h; #define a POSIX macro to import mlockall flags. 2003-07-29 * version 1.1.7 * big bug fix: item exptime 0 meant expire immediately, not never * version 1.1.8 2003-07-22 * make 'delete' take second arg, of time to refuse new add/replace * set/add/replace/delete can all take abs or delta time (delta can't be larger than a month) 2003-07-21 * added doc/protocol.txt 2003-07-01 * report CPU usage in stats 2003-06-30 * version 1.1.6 * fix a number of obscure bugs * more stats reporting 2003-06-10 * removing use of Judy; use a hash. (judy caused memory fragmentation) * shrink some structures * security improvements * version 1.1.0 2003-06-18 * changing maxsize back to an unsigned int 2003-06-16 * adding PHP support * added CONTRIBUTORS file * version 1.0.4 2003-06-15 * forgot to distribute website/api (still learning auto*) * version 1.0.3 2003-06-15 * update to version 1.0.2 * autoconf/automake fixes for older versions * make stats report version number * change license from GPL to BSD Fri, 13 Jun 2003 10:05:51 -0700 Evan Martin * configure.ac, autogen.sh, Makefile.am: Use autotools. * items.c, memcached.c: #include for time(), printf time_t as %lu (is this correct?), minor warnings fixes. memcached-1.5.22/restart.c0000664000175000017500000003324513611703161012270 00000000000000#include "memcached.h" #include "restart.h" #include #include #include #include #include #include #include typedef struct _restart_data_cb restart_data_cb; struct _restart_data_cb { void *data; // user supplied opaque data. struct _restart_data_cb *next; // callbacks are ordered stack restart_check_cb ccb; restart_save_cb scb; char tag[RESTART_TAG_MAXLEN]; }; // TODO: struct to hand back to caller. static int mmap_fd = 0; static void *mmap_base = NULL; static size_t slabmem_limit = 0; char *memory_file = NULL; static restart_data_cb *cb_stack = NULL; // Allows submodules and engines to have independent check and save metadata // routines for the restart code. void restart_register(const char *tag, restart_check_cb ccb, restart_save_cb scb, void *data) { restart_data_cb *cb = calloc(1, sizeof(restart_data_cb)); if (cb == NULL) { fprintf(stderr, "[restart] failed to allocate callback register\n"); abort(); } // Handle first time call initialization inline so we don't need separate // API call. if (cb_stack == NULL) { cb_stack = cb; } else { // Ensure we fire the callbacks in registration order. // Someday I'll get a queue.h overhaul. restart_data_cb *finder = cb_stack; while (finder->next != NULL) { finder = finder->next; } finder->next = cb; } safe_strcpy(cb->tag, tag, RESTART_TAG_MAXLEN); cb->data = data; cb->ccb = *ccb; cb->scb = *scb; } typedef struct { FILE *f; restart_data_cb *cb; char *line; bool done; } restart_cb_ctx; // TODO: error string from cb? // - look for final line with checksum // - checksum entire file (up until final line) // - seek to start static int restart_check(const char *file) { // metadata is kept in a separate file. size_t flen = strlen(file); const char *ext = ".meta"; char *metafile = calloc(1, flen + strlen(ext) + 1); if (metafile == NULL) { // probably in a really bad position if we hit here, so don't start. fprintf(stderr, "[restart] failed to allocate memory for restart check\n"); abort(); } memcpy(metafile, file, flen); memcpy(metafile+flen, ext, strlen(ext)); FILE *f = fopen(metafile, "r"); if (f == NULL) { fprintf(stderr, "[restart] no metadata save file, starting with a clean cache\n"); free(metafile); return -1; } restart_cb_ctx ctx; ctx.f = f; ctx.cb = NULL; ctx.line = NULL; ctx.done = false; if (restart_get_kv(&ctx, NULL, NULL) != RESTART_DONE) { // First line must be a tag, so read it in and set up the proper // callback here. fprintf(stderr, "[restart] corrupt metadata file\n"); // TODO: this should probably just return -1 and skip the reuse. abort(); } if (ctx.cb == NULL) { fprintf(stderr, "[restart] Failed to read a tag from metadata file\n"); abort(); } // loop call the callback, check result code. bool failed = false; while (!ctx.done) { restart_data_cb *cb = ctx.cb; if (cb->ccb(cb->tag, &ctx, cb->data) != 0) { failed = true; break; } } if (ctx.line) free(ctx.line); fclose(f); unlink(metafile); free(metafile); if (failed) { fprintf(stderr, "[restart] failed to valiate metadata, starting with a clean cache\n"); return -1; } else { return 0; } } // This function advances the file read while being called directly from the // callback. // The control inversion here (callback calling in which might change the next // callback) allows the callbacks to set up proper loops or sequences for // reading data back, avoiding an event model. enum restart_get_kv_ret restart_get_kv(void *ctx, char **key, char **val) { char *line = NULL; size_t len = 0; restart_data_cb *cb = NULL; restart_cb_ctx *c = (restart_cb_ctx *) ctx; // free previous line. // we could just pass it into getline, but it can randomly realloc so we'd // have to re-assign it into the structure anyway. if (c->line != NULL) { free(c->line); } if (getline(&line, &len, c->f) != -1) { // First char is an indicator: // T for TAG, changing the callback we use. // K for key/value, to ship to the active callback. char *p = line; while (*p != '\n') { p++; } *p = '\0'; if (line[0] == 'T') { cb = cb_stack; while (cb != NULL) { // NOTE: len is allocated size, not line len. need to chomp \n if (strcmp(cb->tag, line+1) == 0) { break; } cb = cb->next; } if (cb == NULL) { fprintf(stderr, "[restart] internal handler for metadata tag not found: %s:\n", line+1); return RESTART_NOTAG; } c->cb = cb; } else if (line[0] == 'K') { char *p = line+1; // start just ahead of the token. // tokenize the string and return the pointers? if (key != NULL) { *key = p; } // turn key into a normal NULL terminated string. while (*p != ' ' && (p - line < len)) { p++; } *p = '\0'; p++; // value _should_ run until where the newline was, which is \0 now if (val != NULL) { *val = p; } c->line = line; return RESTART_OK; } else { // FIXME: proper error chain. fprintf(stderr, "[restart] invalid metadata line:\n\n%s\n", line); return RESTART_BADLINE; } } else { // EOF or error in read. c->done = true; } return RESTART_DONE; } // TODO: // - rolling checksum along with the writes. // - write final line + checksum + byte count or w/e. static int restart_save(const char *file) { // metadata is kept in a separate file. // FIXME: function. size_t flen = strlen(file); const char *ext = ".meta"; size_t extlen = strlen(ext); char *metafile = calloc(1, flen + extlen + 1); if (metafile == NULL) { fprintf(stderr, "[restart] failed to allocate memory during metadata save\n"); return -1; } memcpy(metafile, file, flen); memcpy(metafile+flen, ext, extlen); // restrictive permissions for the metadata file. // TODO: also for the mmap file eh? :P mode_t oldmask = umask(~(S_IRUSR | S_IWUSR)); FILE *f = fopen(metafile, "w"); umask(oldmask); if (f == NULL) { // FIXME: correct error handling. free(metafile); perror("failed to write metadata file"); return -1; } restart_data_cb *cb = cb_stack; restart_cb_ctx ctx; ctx.f = f; while (cb != NULL) { // Plugins/engines in the metadata file are separated by tag lines. fprintf(f, "T%s\n", cb->tag); if (cb->scb(cb->tag, &ctx, cb->data) != 0) { fclose(f); free(metafile); return -1; } cb = cb->next; } fclose(f); free(metafile); return 0; } // Keys and values must not contain spaces or newlines. // Could offer an interface that uriencodes values for the caller, however // nothing currently would use it, so add when necessary. #define SET_VAL_MAX 4096 void restart_set_kv(void *ctx, const char *key, const char *fmt, ...) { va_list ap; restart_cb_ctx *c = (restart_cb_ctx *) ctx; char valbuf[SET_VAL_MAX]; va_start(ap, fmt); int vlen = vsnprintf(valbuf, SET_VAL_MAX-1, fmt, ap); va_end(ap); // This is heavy handed. We need to protect against corrupt data as much // as possible. The buffer is large and these values are currently small, // it will take a significant mistake to land here. if (vlen >= SET_VAL_MAX) { fprintf(stderr, "[restart] fatal error while saving metadata state, value too long for: %s %s", key, valbuf); abort(); } fprintf(c->f, "K%s %s\n", key, valbuf); // TODO: update crc32c } static long _find_pagesize(void) { #if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE) return sysconf(_SC_PAGESIZE); #else // A good guess. return 4096; #endif } bool restart_mmap_open(const size_t limit, const char *file, void **mem_base) { bool reuse_mmap = true; long pagesize = _find_pagesize(); memory_file = strdup(file); mmap_fd = open(file, O_RDWR|O_CREAT, S_IRWXU); if (mmap_fd == -1) { perror("failed to open file for mmap"); abort(); } if (ftruncate(mmap_fd, limit) != 0) { perror("ftruncate failed"); abort(); } /* Allocate everything in a big chunk with malloc */ if (limit % pagesize) { // This is a sanity check; shouldn't ever be possible since we // increase memory by whole megabytes. fprintf(stderr, "[restart] memory limit not divisible evenly by pagesize (please report bug)\n"); abort(); } mmap_base = mmap(NULL, limit, PROT_READ|PROT_WRITE, MAP_SHARED, mmap_fd, 0); if (mmap_base == MAP_FAILED) { perror("failed to mmap, aborting"); abort(); } // Set the limit before calling check_mmap, so we can find the meta page.. slabmem_limit = limit; if (restart_check(file) != 0) { reuse_mmap = false; } *mem_base = mmap_base; return reuse_mmap; } /* Gracefully stop/close the shared memory segment */ void restart_mmap_close(void) { msync(mmap_base, slabmem_limit, MS_SYNC); if (restart_save(memory_file) != 0) { fprintf(stderr, "[restart] failed to save metadata"); } if (munmap(mmap_base, slabmem_limit) != 0) { perror("[restart] failed to munmap shared memory"); } else if (close(mmap_fd) != 0) { perror("[restart] failed to close shared memory fd"); } free(memory_file); } // given memory base, quickly walk memory and do pointer fixup. // do this once on startup to avoid having to do pointer fixup on every // reference from hash table or LRU. unsigned int restart_fixup(void *orig_addr) { struct timeval tv; uint64_t checked = 0; const unsigned int page_size = settings.slab_page_size; unsigned int page_remain = page_size; gettimeofday(&tv, NULL); if (settings.verbose > 0) { fprintf(stderr, "[restart] original memory base: [%p] new base: [%p]\n", orig_addr, mmap_base); fprintf(stderr, "[restart] recovery start [%d.%d]\n", (int)tv.tv_sec, (int)tv.tv_usec); } // since chunks don't align with pages, we have to also track page size. while (checked < slabmem_limit) { //fprintf(stderr, "checked: %lu\n", checked); item *it = (item *)((char *)mmap_base + checked); int size = slabs_fixup((char *)mmap_base + checked, checked % settings.slab_page_size); //fprintf(stderr, "id: %d, size: %d\n", it->slabs_clsid, size); // slabber gobbled an entire page, skip and move on. if (size == -1) { assert(page_remain % page_size == 0); assert(page_remain == page_size); checked += page_remain; page_remain = page_size; continue; } if (it->it_flags & ITEM_LINKED) { // fixup next/prev links while on LRU. if (it->next) { it->next = (item *)((mc_ptr_t)it->next - (mc_ptr_t)orig_addr); it->next = (item *)((mc_ptr_t)it->next + (mc_ptr_t)mmap_base); } if (it->prev) { it->prev = (item *)((mc_ptr_t)it->prev - (mc_ptr_t)orig_addr); it->prev = (item *)((mc_ptr_t)it->prev + (mc_ptr_t)mmap_base); } //fprintf(stderr, "item was linked\n"); do_item_link_fixup(it); } if (it->it_flags & (ITEM_CHUNKED|ITEM_CHUNK)) { item_chunk *ch; if (it->it_flags & ITEM_CHUNKED) { ch = (item_chunk *) ITEM_schunk(it); // Sigh. Chunked items are a hack; the clsid is the clsid of // the full object (always the largest slab class) rather than // the actual chunk. // I bet this is fixable :( size = slabs_size(ch->orig_clsid); //fprintf(stderr, "fixing chunked item header [%d]\n", size); } else { //fprintf(stderr, "fixing item chunk [%d]\n", size); ch = (item_chunk *) it; } if (ch->next) { ch->next = (item_chunk *)((mc_ptr_t)ch->next - (mc_ptr_t)orig_addr); ch->next = (item_chunk *)((mc_ptr_t)ch->next + (mc_ptr_t)mmap_base); } if (ch->prev) { ch->prev = (item_chunk *)((mc_ptr_t)ch->prev - (mc_ptr_t)orig_addr); ch->prev = (item_chunk *)((mc_ptr_t)ch->prev + (mc_ptr_t)mmap_base); } if (ch->head) { ch->head = (item *)((mc_ptr_t)ch->head - (mc_ptr_t)orig_addr); ch->head = (item *)((mc_ptr_t)ch->head + (mc_ptr_t)mmap_base); } } // next chunk checked += size; page_remain -= size; if (size > page_remain) { //fprintf(stderr, "doot %d\n", page_remain); checked += page_remain; page_remain = settings.slab_page_size; } //assert(checked != 3145728); } if (settings.verbose > 0) { gettimeofday(&tv, NULL); fprintf(stderr, "[restart] recovery end [%d.%d]\n", (int)tv.tv_sec, (int)tv.tv_usec); } return 0; } memcached-1.5.22/hash.h0000664000175000017500000000035513615376577011555 00000000000000#ifndef HASH_H #define HASH_H typedef uint32_t (*hash_func)(const void *key, size_t length); hash_func hash; enum hashfunc_type { JENKINS_HASH=0, MURMUR3_HASH }; int hash_init(enum hashfunc_type type); #endif /* HASH_H */ memcached-1.5.22/aclocal.m40000664000175000017500000012422413615454042012303 00000000000000# generated automatically by aclocal 1.15.1 -*- Autoconf -*- # Copyright (C) 1996-2017 Free Software Foundation, Inc. # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY, to the extent permitted by law; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR A # PARTICULAR PURPOSE. m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) m4_ifndef([AC_AUTOCONF_VERSION], [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.69],, [m4_warning([this file was generated for autoconf 2.69. You have another version of autoconf. It may work, but is not guaranteed to. If you have problems, you may need to regenerate the build system entirely. To do so, use the procedure documented by the package, typically 'autoreconf'.])]) # Copyright (C) 2002-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_AUTOMAKE_VERSION(VERSION) # ---------------------------- # Automake X.Y traces this macro to ensure aclocal.m4 has been # generated from the m4 files accompanying Automake X.Y. # (This private macro should not be called outside this file.) AC_DEFUN([AM_AUTOMAKE_VERSION], [am__api_version='1.15' dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to dnl require some minimum version. Point them to the right macro. m4_if([$1], [1.15.1], [], [AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl ]) # _AM_AUTOCONF_VERSION(VERSION) # ----------------------------- # aclocal traces this macro to find the Autoconf version. # This is a private macro too. Using m4_define simplifies # the logic in aclocal, which can simply ignore this definition. m4_define([_AM_AUTOCONF_VERSION], []) # AM_SET_CURRENT_AUTOMAKE_VERSION # ------------------------------- # Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced. # This function is AC_REQUIREd by AM_INIT_AUTOMAKE. AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION], [AM_AUTOMAKE_VERSION([1.15.1])dnl m4_ifndef([AC_AUTOCONF_VERSION], [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl _AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))]) # AM_AUX_DIR_EXPAND -*- Autoconf -*- # Copyright (C) 2001-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # For projects using AC_CONFIG_AUX_DIR([foo]), Autoconf sets # $ac_aux_dir to '$srcdir/foo'. In other projects, it is set to # '$srcdir', '$srcdir/..', or '$srcdir/../..'. # # Of course, Automake must honor this variable whenever it calls a # tool from the auxiliary directory. The problem is that $srcdir (and # therefore $ac_aux_dir as well) can be either absolute or relative, # depending on how configure is run. This is pretty annoying, since # it makes $ac_aux_dir quite unusable in subdirectories: in the top # source directory, any form will work fine, but in subdirectories a # relative path needs to be adjusted first. # # $ac_aux_dir/missing # fails when called from a subdirectory if $ac_aux_dir is relative # $top_srcdir/$ac_aux_dir/missing # fails if $ac_aux_dir is absolute, # fails when called from a subdirectory in a VPATH build with # a relative $ac_aux_dir # # The reason of the latter failure is that $top_srcdir and $ac_aux_dir # are both prefixed by $srcdir. In an in-source build this is usually # harmless because $srcdir is '.', but things will broke when you # start a VPATH build or use an absolute $srcdir. # # So we could use something similar to $top_srcdir/$ac_aux_dir/missing, # iff we strip the leading $srcdir from $ac_aux_dir. That would be: # am_aux_dir='\$(top_srcdir)/'`expr "$ac_aux_dir" : "$srcdir//*\(.*\)"` # and then we would define $MISSING as # MISSING="\${SHELL} $am_aux_dir/missing" # This will work as long as MISSING is not called from configure, because # unfortunately $(top_srcdir) has no meaning in configure. # However there are other variables, like CC, which are often used in # configure, and could therefore not use this "fixed" $ac_aux_dir. # # Another solution, used here, is to always expand $ac_aux_dir to an # absolute PATH. The drawback is that using absolute paths prevent a # configured tree to be moved without reconfiguration. AC_DEFUN([AM_AUX_DIR_EXPAND], [AC_REQUIRE([AC_CONFIG_AUX_DIR_DEFAULT])dnl # Expand $ac_aux_dir to an absolute path. am_aux_dir=`cd "$ac_aux_dir" && pwd` ]) # AM_CONDITIONAL -*- Autoconf -*- # Copyright (C) 1997-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_CONDITIONAL(NAME, SHELL-CONDITION) # ------------------------------------- # Define a conditional. AC_DEFUN([AM_CONDITIONAL], [AC_PREREQ([2.52])dnl m4_if([$1], [TRUE], [AC_FATAL([$0: invalid condition: $1])], [$1], [FALSE], [AC_FATAL([$0: invalid condition: $1])])dnl AC_SUBST([$1_TRUE])dnl AC_SUBST([$1_FALSE])dnl _AM_SUBST_NOTMAKE([$1_TRUE])dnl _AM_SUBST_NOTMAKE([$1_FALSE])dnl m4_define([_AM_COND_VALUE_$1], [$2])dnl if $2; then $1_TRUE= $1_FALSE='#' else $1_TRUE='#' $1_FALSE= fi AC_CONFIG_COMMANDS_PRE( [if test -z "${$1_TRUE}" && test -z "${$1_FALSE}"; then AC_MSG_ERROR([[conditional "$1" was never defined. Usually this means the macro was only invoked conditionally.]]) fi])]) # Copyright (C) 1999-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # There are a few dirty hacks below to avoid letting 'AC_PROG_CC' be # written in clear, in which case automake, when reading aclocal.m4, # will think it sees a *use*, and therefore will trigger all it's # C support machinery. Also note that it means that autoscan, seeing # CC etc. in the Makefile, will ask for an AC_PROG_CC use... # _AM_DEPENDENCIES(NAME) # ---------------------- # See how the compiler implements dependency checking. # NAME is "CC", "CXX", "OBJC", "OBJCXX", "UPC", or "GJC". # We try a few techniques and use that to set a single cache variable. # # We don't AC_REQUIRE the corresponding AC_PROG_CC since the latter was # modified to invoke _AM_DEPENDENCIES(CC); we would have a circular # dependency, and given that the user is not expected to run this macro, # just rely on AC_PROG_CC. AC_DEFUN([_AM_DEPENDENCIES], [AC_REQUIRE([AM_SET_DEPDIR])dnl AC_REQUIRE([AM_OUTPUT_DEPENDENCY_COMMANDS])dnl AC_REQUIRE([AM_MAKE_INCLUDE])dnl AC_REQUIRE([AM_DEP_TRACK])dnl m4_if([$1], [CC], [depcc="$CC" am_compiler_list=], [$1], [CXX], [depcc="$CXX" am_compiler_list=], [$1], [OBJC], [depcc="$OBJC" am_compiler_list='gcc3 gcc'], [$1], [OBJCXX], [depcc="$OBJCXX" am_compiler_list='gcc3 gcc'], [$1], [UPC], [depcc="$UPC" am_compiler_list=], [$1], [GCJ], [depcc="$GCJ" am_compiler_list='gcc3 gcc'], [depcc="$$1" am_compiler_list=]) AC_CACHE_CHECK([dependency style of $depcc], [am_cv_$1_dependencies_compiler_type], [if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then # We make a subdir and do the tests there. Otherwise we can end up # making bogus files that we don't know about and never remove. For # instance it was reported that on HP-UX the gcc test will end up # making a dummy file named 'D' -- because '-MD' means "put the output # in D". rm -rf conftest.dir mkdir conftest.dir # Copy depcomp to subdir because otherwise we won't find it if we're # using a relative directory. cp "$am_depcomp" conftest.dir cd conftest.dir # We will build objects and dependencies in a subdirectory because # it helps to detect inapplicable dependency modes. For instance # both Tru64's cc and ICC support -MD to output dependencies as a # side effect of compilation, but ICC will put the dependencies in # the current directory while Tru64 will put them in the object # directory. mkdir sub am_cv_$1_dependencies_compiler_type=none if test "$am_compiler_list" = ""; then am_compiler_list=`sed -n ['s/^#*\([a-zA-Z0-9]*\))$/\1/p'] < ./depcomp` fi am__universal=false m4_case([$1], [CC], [case " $depcc " in #( *\ -arch\ *\ -arch\ *) am__universal=true ;; esac], [CXX], [case " $depcc " in #( *\ -arch\ *\ -arch\ *) am__universal=true ;; esac]) for depmode in $am_compiler_list; do # Setup a source with many dependencies, because some compilers # like to wrap large dependency lists on column 80 (with \), and # we should not choose a depcomp mode which is confused by this. # # We need to recreate these files for each test, as the compiler may # overwrite some of them when testing with obscure command lines. # This happens at least with the AIX C compiler. : > sub/conftest.c for i in 1 2 3 4 5 6; do echo '#include "conftst'$i'.h"' >> sub/conftest.c # Using ": > sub/conftst$i.h" creates only sub/conftst1.h with # Solaris 10 /bin/sh. echo '/* dummy */' > sub/conftst$i.h done echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf # We check with '-c' and '-o' for the sake of the "dashmstdout" # mode. It turns out that the SunPro C++ compiler does not properly # handle '-M -o', and we need to detect this. Also, some Intel # versions had trouble with output in subdirs. am__obj=sub/conftest.${OBJEXT-o} am__minus_obj="-o $am__obj" case $depmode in gcc) # This depmode causes a compiler race in universal mode. test "$am__universal" = false || continue ;; nosideeffect) # After this tag, mechanisms are not by side-effect, so they'll # only be used when explicitly requested. if test "x$enable_dependency_tracking" = xyes; then continue else break fi ;; msvc7 | msvc7msys | msvisualcpp | msvcmsys) # This compiler won't grok '-c -o', but also, the minuso test has # not run yet. These depmodes are late enough in the game, and # so weak that their functioning should not be impacted. am__obj=conftest.${OBJEXT-o} am__minus_obj= ;; none) break ;; esac if depmode=$depmode \ source=sub/conftest.c object=$am__obj \ depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ >/dev/null 2>conftest.err && grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && grep $am__obj sub/conftest.Po > /dev/null 2>&1 && ${MAKE-make} -s -f confmf > /dev/null 2>&1; then # icc doesn't choke on unknown options, it will just issue warnings # or remarks (even with -Werror). So we grep stderr for any message # that says an option was ignored or not supported. # When given -MP, icc 7.0 and 7.1 complain thusly: # icc: Command line warning: ignoring option '-M'; no argument required # The diagnosis changed in icc 8.0: # icc: Command line remark: option '-MP' not supported if (grep 'ignoring option' conftest.err || grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else am_cv_$1_dependencies_compiler_type=$depmode break fi fi done cd .. rm -rf conftest.dir else am_cv_$1_dependencies_compiler_type=none fi ]) AC_SUBST([$1DEPMODE], [depmode=$am_cv_$1_dependencies_compiler_type]) AM_CONDITIONAL([am__fastdep$1], [ test "x$enable_dependency_tracking" != xno \ && test "$am_cv_$1_dependencies_compiler_type" = gcc3]) ]) # AM_SET_DEPDIR # ------------- # Choose a directory name for dependency files. # This macro is AC_REQUIREd in _AM_DEPENDENCIES. AC_DEFUN([AM_SET_DEPDIR], [AC_REQUIRE([AM_SET_LEADING_DOT])dnl AC_SUBST([DEPDIR], ["${am__leading_dot}deps"])dnl ]) # AM_DEP_TRACK # ------------ AC_DEFUN([AM_DEP_TRACK], [AC_ARG_ENABLE([dependency-tracking], [dnl AS_HELP_STRING( [--enable-dependency-tracking], [do not reject slow dependency extractors]) AS_HELP_STRING( [--disable-dependency-tracking], [speeds up one-time build])]) if test "x$enable_dependency_tracking" != xno; then am_depcomp="$ac_aux_dir/depcomp" AMDEPBACKSLASH='\' am__nodep='_no' fi AM_CONDITIONAL([AMDEP], [test "x$enable_dependency_tracking" != xno]) AC_SUBST([AMDEPBACKSLASH])dnl _AM_SUBST_NOTMAKE([AMDEPBACKSLASH])dnl AC_SUBST([am__nodep])dnl _AM_SUBST_NOTMAKE([am__nodep])dnl ]) # Generate code to set up dependency tracking. -*- Autoconf -*- # Copyright (C) 1999-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # _AM_OUTPUT_DEPENDENCY_COMMANDS # ------------------------------ AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS], [{ # Older Autoconf quotes --file arguments for eval, but not when files # are listed without --file. Let's play safe and only enable the eval # if we detect the quoting. case $CONFIG_FILES in *\'*) eval set x "$CONFIG_FILES" ;; *) set x $CONFIG_FILES ;; esac shift for mf do # Strip MF so we end up with the name of the file. mf=`echo "$mf" | sed -e 's/:.*$//'` # Check whether this is an Automake generated Makefile or not. # We used to match only the files named 'Makefile.in', but # some people rename them; so instead we look at the file content. # Grep'ing the first line is not enough: some people post-process # each Makefile.in and add a new line on top of each file to say so. # Grep'ing the whole file is not good either: AIX grep has a line # limit of 2048, but all sed's we know have understand at least 4000. if sed -n 's,^#.*generated by automake.*,X,p' "$mf" | grep X >/dev/null 2>&1; then dirpart=`AS_DIRNAME("$mf")` else continue fi # Extract the definition of DEPDIR, am__include, and am__quote # from the Makefile without running 'make'. DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` test -z "$DEPDIR" && continue am__include=`sed -n 's/^am__include = //p' < "$mf"` test -z "$am__include" && continue am__quote=`sed -n 's/^am__quote = //p' < "$mf"` # Find all dependency output files, they are included files with # $(DEPDIR) in their names. We invoke sed twice because it is the # simplest approach to changing $(DEPDIR) to its actual value in the # expansion. for file in `sed -n " s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g'`; do # Make sure the directory exists. test -f "$dirpart/$file" && continue fdir=`AS_DIRNAME(["$file"])` AS_MKDIR_P([$dirpart/$fdir]) # echo "creating $dirpart/$file" echo '# dummy' > "$dirpart/$file" done done } ])# _AM_OUTPUT_DEPENDENCY_COMMANDS # AM_OUTPUT_DEPENDENCY_COMMANDS # ----------------------------- # This macro should only be invoked once -- use via AC_REQUIRE. # # This code is only required when automatic dependency tracking # is enabled. FIXME. This creates each '.P' file that we will # need in order to bootstrap the dependency handling code. AC_DEFUN([AM_OUTPUT_DEPENDENCY_COMMANDS], [AC_CONFIG_COMMANDS([depfiles], [test x"$AMDEP_TRUE" != x"" || _AM_OUTPUT_DEPENDENCY_COMMANDS], [AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir"]) ]) # Do all the work for Automake. -*- Autoconf -*- # Copyright (C) 1996-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # This macro actually does too much. Some checks are only needed if # your package does certain things. But this isn't really a big deal. dnl Redefine AC_PROG_CC to automatically invoke _AM_PROG_CC_C_O. m4_define([AC_PROG_CC], m4_defn([AC_PROG_CC]) [_AM_PROG_CC_C_O ]) # AM_INIT_AUTOMAKE(PACKAGE, VERSION, [NO-DEFINE]) # AM_INIT_AUTOMAKE([OPTIONS]) # ----------------------------------------------- # The call with PACKAGE and VERSION arguments is the old style # call (pre autoconf-2.50), which is being phased out. PACKAGE # and VERSION should now be passed to AC_INIT and removed from # the call to AM_INIT_AUTOMAKE. # We support both call styles for the transition. After # the next Automake release, Autoconf can make the AC_INIT # arguments mandatory, and then we can depend on a new Autoconf # release and drop the old call support. AC_DEFUN([AM_INIT_AUTOMAKE], [AC_PREREQ([2.65])dnl dnl Autoconf wants to disallow AM_ names. We explicitly allow dnl the ones we care about. m4_pattern_allow([^AM_[A-Z]+FLAGS$])dnl AC_REQUIRE([AM_SET_CURRENT_AUTOMAKE_VERSION])dnl AC_REQUIRE([AC_PROG_INSTALL])dnl if test "`cd $srcdir && pwd`" != "`pwd`"; then # Use -I$(srcdir) only when $(srcdir) != ., so that make's output # is not polluted with repeated "-I." AC_SUBST([am__isrc], [' -I$(srcdir)'])_AM_SUBST_NOTMAKE([am__isrc])dnl # test to see if srcdir already configured if test -f $srcdir/config.status; then AC_MSG_ERROR([source directory already configured; run "make distclean" there first]) fi fi # test whether we have cygpath if test -z "$CYGPATH_W"; then if (cygpath --version) >/dev/null 2>/dev/null; then CYGPATH_W='cygpath -w' else CYGPATH_W=echo fi fi AC_SUBST([CYGPATH_W]) # Define the identity of the package. dnl Distinguish between old-style and new-style calls. m4_ifval([$2], [AC_DIAGNOSE([obsolete], [$0: two- and three-arguments forms are deprecated.]) m4_ifval([$3], [_AM_SET_OPTION([no-define])])dnl AC_SUBST([PACKAGE], [$1])dnl AC_SUBST([VERSION], [$2])], [_AM_SET_OPTIONS([$1])dnl dnl Diagnose old-style AC_INIT with new-style AM_AUTOMAKE_INIT. m4_if( m4_ifdef([AC_PACKAGE_NAME], [ok]):m4_ifdef([AC_PACKAGE_VERSION], [ok]), [ok:ok],, [m4_fatal([AC_INIT should be called with package and version arguments])])dnl AC_SUBST([PACKAGE], ['AC_PACKAGE_TARNAME'])dnl AC_SUBST([VERSION], ['AC_PACKAGE_VERSION'])])dnl _AM_IF_OPTION([no-define],, [AC_DEFINE_UNQUOTED([PACKAGE], ["$PACKAGE"], [Name of package]) AC_DEFINE_UNQUOTED([VERSION], ["$VERSION"], [Version number of package])])dnl # Some tools Automake needs. AC_REQUIRE([AM_SANITY_CHECK])dnl AC_REQUIRE([AC_ARG_PROGRAM])dnl AM_MISSING_PROG([ACLOCAL], [aclocal-${am__api_version}]) AM_MISSING_PROG([AUTOCONF], [autoconf]) AM_MISSING_PROG([AUTOMAKE], [automake-${am__api_version}]) AM_MISSING_PROG([AUTOHEADER], [autoheader]) AM_MISSING_PROG([MAKEINFO], [makeinfo]) AC_REQUIRE([AM_PROG_INSTALL_SH])dnl AC_REQUIRE([AM_PROG_INSTALL_STRIP])dnl AC_REQUIRE([AC_PROG_MKDIR_P])dnl # For better backward compatibility. To be removed once Automake 1.9.x # dies out for good. For more background, see: # # AC_SUBST([mkdir_p], ['$(MKDIR_P)']) # We need awk for the "check" target (and possibly the TAP driver). The # system "awk" is bad on some platforms. AC_REQUIRE([AC_PROG_AWK])dnl AC_REQUIRE([AC_PROG_MAKE_SET])dnl AC_REQUIRE([AM_SET_LEADING_DOT])dnl _AM_IF_OPTION([tar-ustar], [_AM_PROG_TAR([ustar])], [_AM_IF_OPTION([tar-pax], [_AM_PROG_TAR([pax])], [_AM_PROG_TAR([v7])])]) _AM_IF_OPTION([no-dependencies],, [AC_PROVIDE_IFELSE([AC_PROG_CC], [_AM_DEPENDENCIES([CC])], [m4_define([AC_PROG_CC], m4_defn([AC_PROG_CC])[_AM_DEPENDENCIES([CC])])])dnl AC_PROVIDE_IFELSE([AC_PROG_CXX], [_AM_DEPENDENCIES([CXX])], [m4_define([AC_PROG_CXX], m4_defn([AC_PROG_CXX])[_AM_DEPENDENCIES([CXX])])])dnl AC_PROVIDE_IFELSE([AC_PROG_OBJC], [_AM_DEPENDENCIES([OBJC])], [m4_define([AC_PROG_OBJC], m4_defn([AC_PROG_OBJC])[_AM_DEPENDENCIES([OBJC])])])dnl AC_PROVIDE_IFELSE([AC_PROG_OBJCXX], [_AM_DEPENDENCIES([OBJCXX])], [m4_define([AC_PROG_OBJCXX], m4_defn([AC_PROG_OBJCXX])[_AM_DEPENDENCIES([OBJCXX])])])dnl ]) AC_REQUIRE([AM_SILENT_RULES])dnl dnl The testsuite driver may need to know about EXEEXT, so add the dnl 'am__EXEEXT' conditional if _AM_COMPILER_EXEEXT was seen. This dnl macro is hooked onto _AC_COMPILER_EXEEXT early, see below. AC_CONFIG_COMMANDS_PRE(dnl [m4_provide_if([_AM_COMPILER_EXEEXT], [AM_CONDITIONAL([am__EXEEXT], [test -n "$EXEEXT"])])])dnl # POSIX will say in a future version that running "rm -f" with no argument # is OK; and we want to be able to make that assumption in our Makefile # recipes. So use an aggressive probe to check that the usage we want is # actually supported "in the wild" to an acceptable degree. # See automake bug#10828. # To make any issue more visible, cause the running configure to be aborted # by default if the 'rm' program in use doesn't match our expectations; the # user can still override this though. if rm -f && rm -fr && rm -rf; then : OK; else cat >&2 <<'END' Oops! Your 'rm' program seems unable to run without file operands specified on the command line, even when the '-f' option is present. This is contrary to the behaviour of most rm programs out there, and not conforming with the upcoming POSIX standard: Please tell bug-automake@gnu.org about your system, including the value of your $PATH and any error possibly output before this message. This can help us improve future automake versions. END if test x"$ACCEPT_INFERIOR_RM_PROGRAM" = x"yes"; then echo 'Configuration will proceed anyway, since you have set the' >&2 echo 'ACCEPT_INFERIOR_RM_PROGRAM variable to "yes"' >&2 echo >&2 else cat >&2 <<'END' Aborting the configuration process, to ensure you take notice of the issue. You can download and install GNU coreutils to get an 'rm' implementation that behaves properly: . If you want to complete the configuration process using your problematic 'rm' anyway, export the environment variable ACCEPT_INFERIOR_RM_PROGRAM to "yes", and re-run configure. END AC_MSG_ERROR([Your 'rm' program is bad, sorry.]) fi fi dnl The trailing newline in this macro's definition is deliberate, for dnl backward compatibility and to allow trailing 'dnl'-style comments dnl after the AM_INIT_AUTOMAKE invocation. See automake bug#16841. ]) dnl Hook into '_AC_COMPILER_EXEEXT' early to learn its expansion. Do not dnl add the conditional right here, as _AC_COMPILER_EXEEXT may be further dnl mangled by Autoconf and run in a shell conditional statement. m4_define([_AC_COMPILER_EXEEXT], m4_defn([_AC_COMPILER_EXEEXT])[m4_provide([_AM_COMPILER_EXEEXT])]) # When config.status generates a header, we must update the stamp-h file. # This file resides in the same directory as the config header # that is generated. The stamp files are numbered to have different names. # Autoconf calls _AC_AM_CONFIG_HEADER_HOOK (when defined) in the # loop where config.status creates the headers, so we can generate # our stamp files there. AC_DEFUN([_AC_AM_CONFIG_HEADER_HOOK], [# Compute $1's index in $config_headers. _am_arg=$1 _am_stamp_count=1 for _am_header in $config_headers :; do case $_am_header in $_am_arg | $_am_arg:* ) break ;; * ) _am_stamp_count=`expr $_am_stamp_count + 1` ;; esac done echo "timestamp for $_am_arg" >`AS_DIRNAME(["$_am_arg"])`/stamp-h[]$_am_stamp_count]) # Copyright (C) 2001-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_PROG_INSTALL_SH # ------------------ # Define $install_sh. AC_DEFUN([AM_PROG_INSTALL_SH], [AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl if test x"${install_sh+set}" != xset; then case $am_aux_dir in *\ * | *\ *) install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;; *) install_sh="\${SHELL} $am_aux_dir/install-sh" esac fi AC_SUBST([install_sh])]) # Copyright (C) 2003-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # Check whether the underlying file-system supports filenames # with a leading dot. For instance MS-DOS doesn't. AC_DEFUN([AM_SET_LEADING_DOT], [rm -rf .tst 2>/dev/null mkdir .tst 2>/dev/null if test -d .tst; then am__leading_dot=. else am__leading_dot=_ fi rmdir .tst 2>/dev/null AC_SUBST([am__leading_dot])]) # Check to see how 'make' treats includes. -*- Autoconf -*- # Copyright (C) 2001-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_MAKE_INCLUDE() # ----------------- # Check to see how make treats includes. AC_DEFUN([AM_MAKE_INCLUDE], [am_make=${MAKE-make} cat > confinc << 'END' am__doit: @echo this is the am__doit target .PHONY: am__doit END # If we don't find an include directive, just comment out the code. AC_MSG_CHECKING([for style of include used by $am_make]) am__include="#" am__quote= _am_result=none # First try GNU make style include. echo "include confinc" > confmf # Ignore all kinds of additional output from 'make'. case `$am_make -s -f confmf 2> /dev/null` in #( *the\ am__doit\ target*) am__include=include am__quote= _am_result=GNU ;; esac # Now try BSD make style include. if test "$am__include" = "#"; then echo '.include "confinc"' > confmf case `$am_make -s -f confmf 2> /dev/null` in #( *the\ am__doit\ target*) am__include=.include am__quote="\"" _am_result=BSD ;; esac fi AC_SUBST([am__include]) AC_SUBST([am__quote]) AC_MSG_RESULT([$_am_result]) rm -f confinc confmf ]) # Fake the existence of programs that GNU maintainers use. -*- Autoconf -*- # Copyright (C) 1997-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_MISSING_PROG(NAME, PROGRAM) # ------------------------------ AC_DEFUN([AM_MISSING_PROG], [AC_REQUIRE([AM_MISSING_HAS_RUN]) $1=${$1-"${am_missing_run}$2"} AC_SUBST($1)]) # AM_MISSING_HAS_RUN # ------------------ # Define MISSING if not defined so far and test if it is modern enough. # If it is, set am_missing_run to use it, otherwise, to nothing. AC_DEFUN([AM_MISSING_HAS_RUN], [AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl AC_REQUIRE_AUX_FILE([missing])dnl if test x"${MISSING+set}" != xset; then case $am_aux_dir in *\ * | *\ *) MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;; *) MISSING="\${SHELL} $am_aux_dir/missing" ;; esac fi # Use eval to expand $SHELL if eval "$MISSING --is-lightweight"; then am_missing_run="$MISSING " else am_missing_run= AC_MSG_WARN(['missing' script is too old or missing]) fi ]) # -*- Autoconf -*- # Obsolete and "removed" macros, that must however still report explicit # error messages when used, to smooth transition. # # Copyright (C) 1996-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. AC_DEFUN([AM_CONFIG_HEADER], [AC_DIAGNOSE([obsolete], ['$0': this macro is obsolete. You should use the 'AC][_CONFIG_HEADERS' macro instead.])dnl AC_CONFIG_HEADERS($@)]) AC_DEFUN([AM_PROG_CC_STDC], [AC_PROG_CC am_cv_prog_cc_stdc=$ac_cv_prog_cc_stdc AC_DIAGNOSE([obsolete], ['$0': this macro is obsolete. You should simply use the 'AC][_PROG_CC' macro instead. Also, your code should no longer depend upon 'am_cv_prog_cc_stdc', but upon 'ac_cv_prog_cc_stdc'.])]) AC_DEFUN([AM_C_PROTOTYPES], [AC_FATAL([automatic de-ANSI-fication support has been removed])]) AU_DEFUN([fp_C_PROTOTYPES], [AM_C_PROTOTYPES]) # Helper functions for option handling. -*- Autoconf -*- # Copyright (C) 2001-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # _AM_MANGLE_OPTION(NAME) # ----------------------- AC_DEFUN([_AM_MANGLE_OPTION], [[_AM_OPTION_]m4_bpatsubst($1, [[^a-zA-Z0-9_]], [_])]) # _AM_SET_OPTION(NAME) # -------------------- # Set option NAME. Presently that only means defining a flag for this option. AC_DEFUN([_AM_SET_OPTION], [m4_define(_AM_MANGLE_OPTION([$1]), [1])]) # _AM_SET_OPTIONS(OPTIONS) # ------------------------ # OPTIONS is a space-separated list of Automake options. AC_DEFUN([_AM_SET_OPTIONS], [m4_foreach_w([_AM_Option], [$1], [_AM_SET_OPTION(_AM_Option)])]) # _AM_IF_OPTION(OPTION, IF-SET, [IF-NOT-SET]) # ------------------------------------------- # Execute IF-SET if OPTION is set, IF-NOT-SET otherwise. AC_DEFUN([_AM_IF_OPTION], [m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])]) # Copyright (C) 1999-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # _AM_PROG_CC_C_O # --------------- # Like AC_PROG_CC_C_O, but changed for automake. We rewrite AC_PROG_CC # to automatically call this. AC_DEFUN([_AM_PROG_CC_C_O], [AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl AC_REQUIRE_AUX_FILE([compile])dnl AC_LANG_PUSH([C])dnl AC_CACHE_CHECK( [whether $CC understands -c and -o together], [am_cv_prog_cc_c_o], [AC_LANG_CONFTEST([AC_LANG_PROGRAM([])]) # Make sure it works both with $CC and with simple cc. # Following AC_PROG_CC_C_O, we do the test twice because some # compilers refuse to overwrite an existing .o file with -o, # though they will create one. am_cv_prog_cc_c_o=yes for am_i in 1 2; do if AM_RUN_LOG([$CC -c conftest.$ac_ext -o conftest2.$ac_objext]) \ && test -f conftest2.$ac_objext; then : OK else am_cv_prog_cc_c_o=no break fi done rm -f core conftest* unset am_i]) if test "$am_cv_prog_cc_c_o" != yes; then # Losing compiler, so override with the script. # FIXME: It is wrong to rewrite CC. # But if we don't then we get into trouble of one sort or another. # A longer-term fix would be to have automake use am__CC in this case, # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" CC="$am_aux_dir/compile $CC" fi AC_LANG_POP([C])]) # For backward compatibility. AC_DEFUN_ONCE([AM_PROG_CC_C_O], [AC_REQUIRE([AC_PROG_CC])]) # Copyright (C) 2001-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_RUN_LOG(COMMAND) # ------------------- # Run COMMAND, save the exit status in ac_status, and log it. # (This has been adapted from Autoconf's _AC_RUN_LOG macro.) AC_DEFUN([AM_RUN_LOG], [{ echo "$as_me:$LINENO: $1" >&AS_MESSAGE_LOG_FD ($1) >&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD ac_status=$? echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD (exit $ac_status); }]) # Check to make sure that the build environment is sane. -*- Autoconf -*- # Copyright (C) 1996-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_SANITY_CHECK # --------------- AC_DEFUN([AM_SANITY_CHECK], [AC_MSG_CHECKING([whether build environment is sane]) # Reject unsafe characters in $srcdir or the absolute working directory # name. Accept space and tab only in the latter. am_lf=' ' case `pwd` in *[[\\\"\#\$\&\'\`$am_lf]]*) AC_MSG_ERROR([unsafe absolute working directory name]);; esac case $srcdir in *[[\\\"\#\$\&\'\`$am_lf\ \ ]]*) AC_MSG_ERROR([unsafe srcdir value: '$srcdir']);; esac # Do 'set' in a subshell so we don't clobber the current shell's # arguments. Must try -L first in case configure is actually a # symlink; some systems play weird games with the mod time of symlinks # (eg FreeBSD returns the mod time of the symlink's containing # directory). if ( am_has_slept=no for am_try in 1 2; do echo "timestamp, slept: $am_has_slept" > conftest.file set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null` if test "$[*]" = "X"; then # -L didn't work. set X `ls -t "$srcdir/configure" conftest.file` fi if test "$[*]" != "X $srcdir/configure conftest.file" \ && test "$[*]" != "X conftest.file $srcdir/configure"; then # If neither matched, then we have a broken ls. This can happen # if, for instance, CONFIG_SHELL is bash and it inherits a # broken ls alias from the environment. This has actually # happened. Such a system could not be considered "sane". AC_MSG_ERROR([ls -t appears to fail. Make sure there is not a broken alias in your environment]) fi if test "$[2]" = conftest.file || test $am_try -eq 2; then break fi # Just in case. sleep 1 am_has_slept=yes done test "$[2]" = conftest.file ) then # Ok. : else AC_MSG_ERROR([newly created file is older than distributed files! Check your system clock]) fi AC_MSG_RESULT([yes]) # If we didn't sleep, we still need to ensure time stamps of config.status and # generated files are strictly newer. am_sleep_pid= if grep 'slept: no' conftest.file >/dev/null 2>&1; then ( sleep 1 ) & am_sleep_pid=$! fi AC_CONFIG_COMMANDS_PRE( [AC_MSG_CHECKING([that generated files are newer than configure]) if test -n "$am_sleep_pid"; then # Hide warnings about reused PIDs. wait $am_sleep_pid 2>/dev/null fi AC_MSG_RESULT([done])]) rm -f conftest.file ]) # Copyright (C) 2009-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_SILENT_RULES([DEFAULT]) # -------------------------- # Enable less verbose build rules; with the default set to DEFAULT # ("yes" being less verbose, "no" or empty being verbose). AC_DEFUN([AM_SILENT_RULES], [AC_ARG_ENABLE([silent-rules], [dnl AS_HELP_STRING( [--enable-silent-rules], [less verbose build output (undo: "make V=1")]) AS_HELP_STRING( [--disable-silent-rules], [verbose build output (undo: "make V=0")])dnl ]) case $enable_silent_rules in @%:@ ((( yes) AM_DEFAULT_VERBOSITY=0;; no) AM_DEFAULT_VERBOSITY=1;; *) AM_DEFAULT_VERBOSITY=m4_if([$1], [yes], [0], [1]);; esac dnl dnl A few 'make' implementations (e.g., NonStop OS and NextStep) dnl do not support nested variable expansions. dnl See automake bug#9928 and bug#10237. am_make=${MAKE-make} AC_CACHE_CHECK([whether $am_make supports nested variables], [am_cv_make_support_nested_variables], [if AS_ECHO([['TRUE=$(BAR$(V)) BAR0=false BAR1=true V=1 am__doit: @$(TRUE) .PHONY: am__doit']]) | $am_make -f - >/dev/null 2>&1; then am_cv_make_support_nested_variables=yes else am_cv_make_support_nested_variables=no fi]) if test $am_cv_make_support_nested_variables = yes; then dnl Using '$V' instead of '$(V)' breaks IRIX make. AM_V='$(V)' AM_DEFAULT_V='$(AM_DEFAULT_VERBOSITY)' else AM_V=$AM_DEFAULT_VERBOSITY AM_DEFAULT_V=$AM_DEFAULT_VERBOSITY fi AC_SUBST([AM_V])dnl AM_SUBST_NOTMAKE([AM_V])dnl AC_SUBST([AM_DEFAULT_V])dnl AM_SUBST_NOTMAKE([AM_DEFAULT_V])dnl AC_SUBST([AM_DEFAULT_VERBOSITY])dnl AM_BACKSLASH='\' AC_SUBST([AM_BACKSLASH])dnl _AM_SUBST_NOTMAKE([AM_BACKSLASH])dnl ]) # Copyright (C) 2001-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # AM_PROG_INSTALL_STRIP # --------------------- # One issue with vendor 'install' (even GNU) is that you can't # specify the program used to strip binaries. This is especially # annoying in cross-compiling environments, where the build's strip # is unlikely to handle the host's binaries. # Fortunately install-sh will honor a STRIPPROG variable, so we # always use install-sh in "make install-strip", and initialize # STRIPPROG with the value of the STRIP variable (set by the user). AC_DEFUN([AM_PROG_INSTALL_STRIP], [AC_REQUIRE([AM_PROG_INSTALL_SH])dnl # Installed binaries are usually stripped using 'strip' when the user # run "make install-strip". However 'strip' might not be the right # tool to use in cross-compilation environments, therefore Automake # will honor the 'STRIP' environment variable to overrule this program. dnl Don't test for $cross_compiling = yes, because it might be 'maybe'. if test "$cross_compiling" != no; then AC_CHECK_TOOL([STRIP], [strip], :) fi INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s" AC_SUBST([INSTALL_STRIP_PROGRAM])]) # Copyright (C) 2006-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # _AM_SUBST_NOTMAKE(VARIABLE) # --------------------------- # Prevent Automake from outputting VARIABLE = @VARIABLE@ in Makefile.in. # This macro is traced by Automake. AC_DEFUN([_AM_SUBST_NOTMAKE]) # AM_SUBST_NOTMAKE(VARIABLE) # -------------------------- # Public sister of _AM_SUBST_NOTMAKE. AC_DEFUN([AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE($@)]) # Check how to create a tarball. -*- Autoconf -*- # Copyright (C) 2004-2017 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # _AM_PROG_TAR(FORMAT) # -------------------- # Check how to create a tarball in format FORMAT. # FORMAT should be one of 'v7', 'ustar', or 'pax'. # # Substitute a variable $(am__tar) that is a command # writing to stdout a FORMAT-tarball containing the directory # $tardir. # tardir=directory && $(am__tar) > result.tar # # Substitute a variable $(am__untar) that extract such # a tarball read from stdin. # $(am__untar) < result.tar # AC_DEFUN([_AM_PROG_TAR], [# Always define AMTAR for backward compatibility. Yes, it's still used # in the wild :-( We should find a proper way to deprecate it ... AC_SUBST([AMTAR], ['$${TAR-tar}']) # We'll loop over all known methods to create a tar archive until one works. _am_tools='gnutar m4_if([$1], [ustar], [plaintar]) pax cpio none' m4_if([$1], [v7], [am__tar='$${TAR-tar} chof - "$$tardir"' am__untar='$${TAR-tar} xf -'], [m4_case([$1], [ustar], [# The POSIX 1988 'ustar' format is defined with fixed-size fields. # There is notably a 21 bits limit for the UID and the GID. In fact, # the 'pax' utility can hang on bigger UID/GID (see automake bug#8343 # and bug#13588). am_max_uid=2097151 # 2^21 - 1 am_max_gid=$am_max_uid # The $UID and $GID variables are not portable, so we need to resort # to the POSIX-mandated id(1) utility. Errors in the 'id' calls # below are definitely unexpected, so allow the users to see them # (that is, avoid stderr redirection). am_uid=`id -u || echo unknown` am_gid=`id -g || echo unknown` AC_MSG_CHECKING([whether UID '$am_uid' is supported by ustar format]) if test $am_uid -le $am_max_uid; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) _am_tools=none fi AC_MSG_CHECKING([whether GID '$am_gid' is supported by ustar format]) if test $am_gid -le $am_max_gid; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) _am_tools=none fi], [pax], [], [m4_fatal([Unknown tar format])]) AC_MSG_CHECKING([how to create a $1 tar archive]) # Go ahead even if we have the value already cached. We do so because we # need to set the values for the 'am__tar' and 'am__untar' variables. _am_tools=${am_cv_prog_tar_$1-$_am_tools} for _am_tool in $_am_tools; do case $_am_tool in gnutar) for _am_tar in tar gnutar gtar; do AM_RUN_LOG([$_am_tar --version]) && break done am__tar="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$$tardir"' am__tar_="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$tardir"' am__untar="$_am_tar -xf -" ;; plaintar) # Must skip GNU tar: if it does not support --format= it doesn't create # ustar tarball either. (tar --version) >/dev/null 2>&1 && continue am__tar='tar chf - "$$tardir"' am__tar_='tar chf - "$tardir"' am__untar='tar xf -' ;; pax) am__tar='pax -L -x $1 -w "$$tardir"' am__tar_='pax -L -x $1 -w "$tardir"' am__untar='pax -r' ;; cpio) am__tar='find "$$tardir" -print | cpio -o -H $1 -L' am__tar_='find "$tardir" -print | cpio -o -H $1 -L' am__untar='cpio -i -H $1 -d' ;; none) am__tar=false am__tar_=false am__untar=false ;; esac # If the value was cached, stop now. We just wanted to have am__tar # and am__untar set. test -n "${am_cv_prog_tar_$1}" && break # tar/untar a dummy directory, and stop if the command works. rm -rf conftest.dir mkdir conftest.dir echo GrepMe > conftest.dir/file AM_RUN_LOG([tardir=conftest.dir && eval $am__tar_ >conftest.tar]) rm -rf conftest.dir if test -s conftest.tar; then AM_RUN_LOG([$am__untar /dev/null 2>&1 && break fi done rm -rf conftest.dir AC_CACHE_VAL([am_cv_prog_tar_$1], [am_cv_prog_tar_$1=$_am_tool]) AC_MSG_RESULT([$am_cv_prog_tar_$1])]) AC_SUBST([am__tar]) AC_SUBST([am__untar]) ]) # _AM_PROG_TAR memcached-1.5.22/AUTHORS0000664000175000017500000000010513461170555011505 00000000000000Anatoly Vorobey Brad Fitzpatrick memcached-1.5.22/tls.c0000664000175000017500000001375713611703161011414 00000000000000#include "memcached.h" #ifdef TLS #include "tls.h" #include #include #include #ifndef MAXPATHLEN #define MAXPATHLEN 4096 #endif static pthread_mutex_t ssl_ctx_lock = PTHREAD_MUTEX_INITIALIZER; const unsigned MAX_ERROR_MSG_SIZE = 128; void SSL_LOCK() { pthread_mutex_lock(&(ssl_ctx_lock)); } void SSL_UNLOCK(void) { pthread_mutex_unlock(&(ssl_ctx_lock)); } /* * Reads decrypted data from the underlying BIO read buffers, * which reads from the socket. */ ssize_t ssl_read(conn *c, void *buf, size_t count) { assert (c != NULL); /* TODO : document the state machine interactions for SSL_read with non-blocking sockets/ SSL re-negotiations */ return SSL_read(c->ssl, buf, count); } /* * SSL sendmsg implementation. Perform a SSL_write. */ ssize_t ssl_sendmsg(conn *c, struct msghdr *msg, int flags) { assert (c != NULL); size_t buf_remain = settings.ssl_wbuf_size; size_t bytes = 0; size_t to_copy; int i; // ssl_wbuf is pointing to the buffer allocated in the worker thread. assert(c->ssl_wbuf); // TODO: allocate a fix buffer in crawler/logger if they start using // the sendmsg method. Also, set c->ssl_wbuf when the side thread // start owning the connection and reset the pointer in // conn_worker_readd. // Currntly this connection would not be served by a different thread // than the one it's assigned. assert(pthread_equal(c->thread->thread_id, pthread_self()) != 0); char *bp = c->ssl_wbuf; for (i = 0; i < msg->msg_iovlen; i++) { size_t len = msg->msg_iov[i].iov_len; to_copy = len < buf_remain ? len : buf_remain; memcpy(bp + bytes, (void*)msg->msg_iov[i].iov_base, to_copy); buf_remain -= to_copy; bytes += to_copy; if (buf_remain == 0) break; } /* TODO : document the state machine interactions for SSL_write with non-blocking sockets/ SSL re-negotiations */ return SSL_write(c->ssl, c->ssl_wbuf, bytes); } /* * Writes data to the underlying BIO write buffers, * which encrypt and write them to the socket. */ ssize_t ssl_write(conn *c, void *buf, size_t count) { assert (c != NULL); return SSL_write(c->ssl, buf, count); } /* * Loads server certificates to the SSL context and validate them. * @return whether certificates are successfully loaded and verified or not. * @param error_msg contains the error when unsuccessful. */ static bool load_server_certificates(char **errmsg) { bool success = true; char *error_msg = malloc(MAXPATHLEN + MAX_ERROR_MSG_SIZE); size_t errmax = MAXPATHLEN + MAX_ERROR_MSG_SIZE - 1; if (error_msg == NULL) { *errmsg = NULL; return false; } SSL_LOCK(); if (!SSL_CTX_use_certificate_chain_file(settings.ssl_ctx, settings.ssl_chain_cert)) { snprintf(error_msg, errmax, "Error loading the certificate chain: %s\r\n", settings.ssl_chain_cert); success = false; } else if (!SSL_CTX_use_PrivateKey_file(settings.ssl_ctx, settings.ssl_key, settings.ssl_keyformat)) { snprintf(error_msg, errmax, "Error loading the key: %s\r\n", settings.ssl_key); success = false; } else if (!SSL_CTX_check_private_key(settings.ssl_ctx)) { snprintf(error_msg, errmax, "Error validating the certificate\r\n"); success = false; } else if (settings.ssl_ca_cert) { if (!SSL_CTX_load_verify_locations(settings.ssl_ctx, settings.ssl_ca_cert, NULL)) { snprintf(error_msg, errmax, "Error loading the CA certificate: %s\r\n", settings.ssl_ca_cert); success = false; } else { SSL_CTX_set_client_CA_list(settings.ssl_ctx, SSL_load_client_CA_file(settings.ssl_ca_cert)); } } SSL_UNLOCK(); if (success) { settings.ssl_last_cert_refresh_time = current_time; free(error_msg); } else { *errmsg = error_msg; } return success; } /* * Verify SSL settings and initiates the SSL context. */ int ssl_init(void) { assert(settings.ssl_enabled); // SSL context for the process. All connections will share one // process level context. settings.ssl_ctx = SSL_CTX_new(TLS_server_method()); // Clients should use at least TLSv1.2 int flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_TLSv1 |SSL_OP_NO_TLSv1_1; SSL_CTX_set_options(settings.ssl_ctx, flags); // The server certificate, private key and validations. char *error_msg; if (!load_server_certificates(&error_msg)) { if (settings.verbose) { fprintf(stderr, "%s", error_msg); } free(error_msg); exit(EX_USAGE); } // The verification mode of client certificate, default is SSL_VERIFY_PEER. SSL_CTX_set_verify(settings.ssl_ctx, settings.ssl_verify_mode, NULL); if (settings.ssl_ciphers && !SSL_CTX_set_cipher_list(settings.ssl_ctx, settings.ssl_ciphers)) { if (settings.verbose) { fprintf(stderr, "Error setting the provided cipher(s): %s\n", settings.ssl_ciphers); } exit(EX_USAGE); } return 0; } /* * This method is registered with each SSL connection and abort the SSL session * if a client initiates a renegotiation. * TODO : Proper way to do this is to set SSL_OP_NO_RENEGOTIATION * using the SSL_CTX_set_options but that option only available in * openssl 1.1.0h or above. */ void ssl_callback(const SSL *s, int where, int ret) { SSL* ssl = (SSL*)s; if (SSL_in_before(ssl)) { if (settings.verbose) { fprintf(stderr, "%d: SSL renegotiation is not supported, " "closing the connection\n", SSL_get_fd(ssl)); } SSL_set_shutdown(ssl, SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN); return; } } bool refresh_certs(char **errmsg) { return load_server_certificates(errmsg); } #endif memcached-1.5.22/cache.c0000664000175000017500000000742613615376577011676 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ #include #include #include #ifndef NDEBUG #include #endif #include "cache.h" #ifndef NDEBUG const uint64_t redzone_pattern = 0xdeadbeefcafebabe; int cache_error = 0; #endif const int initial_pool_size = 64; cache_t* cache_create(const char *name, size_t bufsize, size_t align, cache_constructor_t* constructor, cache_destructor_t* destructor) { cache_t* ret = calloc(1, sizeof(cache_t)); char* nm = strdup(name); void** ptr = calloc(initial_pool_size, sizeof(void*)); if (ret == NULL || nm == NULL || ptr == NULL || pthread_mutex_init(&ret->mutex, NULL) == -1) { free(ret); free(nm); free(ptr); return NULL; } ret->name = nm; ret->ptr = ptr; ret->freetotal = initial_pool_size; ret->constructor = constructor; ret->destructor = destructor; #ifndef NDEBUG ret->bufsize = bufsize + 2 * sizeof(redzone_pattern); #else ret->bufsize = bufsize; #endif return ret; } static inline void* get_object(void *ptr) { #ifndef NDEBUG uint64_t *pre = ptr; return pre + 1; #else return ptr; #endif } void cache_destroy(cache_t *cache) { while (cache->freecurr > 0) { void *ptr = cache->ptr[--cache->freecurr]; if (cache->destructor) { cache->destructor(get_object(ptr), NULL); } free(ptr); } free(cache->name); free(cache->ptr); pthread_mutex_destroy(&cache->mutex); free(cache); } void* cache_alloc(cache_t *cache) { void *ret; pthread_mutex_lock(&cache->mutex); ret = do_cache_alloc(cache); pthread_mutex_unlock(&cache->mutex); return ret; } void* do_cache_alloc(cache_t *cache) { void *ret; void *object; if (cache->freecurr > 0) { ret = cache->ptr[--cache->freecurr]; object = get_object(ret); } else { object = ret = malloc(cache->bufsize); if (ret != NULL) { object = get_object(ret); if (cache->constructor != NULL && cache->constructor(object, NULL, 0) != 0) { free(ret); object = NULL; } } } #ifndef NDEBUG if (object != NULL) { /* add a simple form of buffer-check */ uint64_t *pre = ret; *pre = redzone_pattern; ret = pre+1; memcpy(((char*)ret) + cache->bufsize - (2 * sizeof(redzone_pattern)), &redzone_pattern, sizeof(redzone_pattern)); } #endif return object; } void cache_free(cache_t *cache, void *ptr) { pthread_mutex_lock(&cache->mutex); do_cache_free(cache, ptr); pthread_mutex_unlock(&cache->mutex); } void do_cache_free(cache_t *cache, void *ptr) { #ifndef NDEBUG /* validate redzone... */ if (memcmp(((char*)ptr) + cache->bufsize - (2 * sizeof(redzone_pattern)), &redzone_pattern, sizeof(redzone_pattern)) != 0) { raise(SIGABRT); cache_error = 1; return; } uint64_t *pre = ptr; --pre; if (*pre != redzone_pattern) { raise(SIGABRT); cache_error = -1; return; } ptr = pre; #endif if (cache->freecurr < cache->freetotal) { cache->ptr[cache->freecurr++] = ptr; } else { /* try to enlarge free connections array */ size_t newtotal = cache->freetotal * 2; void **new_free = realloc(cache->ptr, sizeof(char *) * newtotal); if (new_free) { cache->freetotal = newtotal; cache->ptr = new_free; cache->ptr[cache->freecurr++] = ptr; } else { if (cache->destructor) { cache->destructor(ptr, NULL); } free(ptr); } } } memcached-1.5.22/NEWS0000664000175000017500000000002713575022500011127 00000000000000https://memcached.org/ memcached-1.5.22/memcached.h0000664000175000017500000007506713615376577012554 00000000000000/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ /** \file * The main memcached header holding commonly used data * structures and function prototypes. */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include "itoa_ljust.h" #include "protocol_binary.h" #include "cache.h" #include "logger.h" #ifdef EXTSTORE #include "extstore.h" #include "crc32c.h" #endif #include "sasl_defs.h" #ifdef TLS #include #endif /** Maximum length of a key. */ #define KEY_MAX_LENGTH 250 /** Maximum length of a uri encoded key. */ #define KEY_MAX_URI_ENCODED_LENGTH (KEY_MAX_LENGTH * 3 + 1) /** Size of an incr buf. */ #define INCR_MAX_STORAGE_LEN 24 #define DATA_BUFFER_SIZE 2048 #define UDP_READ_BUFFER_SIZE 65536 #define UDP_MAX_PAYLOAD_SIZE 1400 #define UDP_HEADER_SIZE 8 #define MAX_SENDBUF_SIZE (256 * 1024 * 1024) /* Up to 3 numbers (2 32bit, 1 64bit), spaces, newlines, null 0 */ #define SUFFIX_SIZE 50 /** Initial size of list of items being returned by "get". */ #define ITEM_LIST_INITIAL 200 /** Initial size of list of CAS suffixes appended to "gets" lines. */ #define SUFFIX_LIST_INITIAL 100 /** Initial size of the sendmsg() scatter/gather array. */ #define IOV_LIST_INITIAL 400 /** Initial number of sendmsg() argument structures to allocate. */ #define MSG_LIST_INITIAL 10 /** High water marks for buffer shrinking */ #define READ_BUFFER_HIGHWAT 8192 #define ITEM_LIST_HIGHWAT 400 #define IOV_LIST_HIGHWAT 600 #define MSG_LIST_HIGHWAT 100 /* Binary protocol stuff */ #define MIN_BIN_PKT_LENGTH 16 #define BIN_PKT_HDR_WORDS (MIN_BIN_PKT_LENGTH/sizeof(uint32_t)) /* Initial power multiplier for the hash table */ #define HASHPOWER_DEFAULT 16 #define HASHPOWER_MAX 32 /* * We only reposition items in the LRU queue if they haven't been repositioned * in this many seconds. That saves us from churning on frequently-accessed * items. */ #define ITEM_UPDATE_INTERVAL 60 /* * Valid range of the maximum size of an item, in bytes. */ #define ITEM_SIZE_MAX_LOWER_LIMIT 1024 #define ITEM_SIZE_MAX_UPPER_LIMIT 1024 * 1024 * 1024 /* unistd.h is here */ #if HAVE_UNISTD_H # include #endif /* Slab sizing definitions. */ #define POWER_SMALLEST 1 #define POWER_LARGEST 256 /* actual cap is 255 */ #define SLAB_GLOBAL_PAGE_POOL 0 /* magic slab class for storing pages for reassignment */ #define CHUNK_ALIGN_BYTES 8 /* slab class max is a 6-bit number, -1. */ #define MAX_NUMBER_OF_SLAB_CLASSES (63 + 1) /** How long an object can reasonably be assumed to be locked before harvesting it on a low memory condition. Default: disabled. */ #define TAIL_REPAIR_TIME_DEFAULT 0 /* warning: don't use these macros with a function, as it evals its arg twice */ #define ITEM_get_cas(i) (((i)->it_flags & ITEM_CAS) ? \ (i)->data->cas : (uint64_t)0) #define ITEM_set_cas(i,v) { \ if ((i)->it_flags & ITEM_CAS) { \ (i)->data->cas = v; \ } \ } #define ITEM_key(item) (((char*)&((item)->data)) \ + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0)) #define ITEM_suffix(item) ((char*) &((item)->data) + (item)->nkey + 1 \ + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0)) #define ITEM_data(item) ((char*) &((item)->data) + (item)->nkey + 1 \ + (((item)->it_flags & ITEM_CFLAGS) ? sizeof(uint32_t) : 0) \ + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0)) #define ITEM_ntotal(item) (sizeof(struct _stritem) + (item)->nkey + 1 \ + (item)->nbytes \ + (((item)->it_flags & ITEM_CFLAGS) ? sizeof(uint32_t) : 0) \ + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0)) #define ITEM_clsid(item) ((item)->slabs_clsid & ~(3<<6)) #define ITEM_lruid(item) ((item)->slabs_clsid & (3<<6)) #define STAT_KEY_LEN 128 #define STAT_VAL_LEN 128 /** Append a simple stat with a stat name, value format and value */ #define APPEND_STAT(name, fmt, val) \ append_stat(name, add_stats, c, fmt, val); /** Append an indexed stat with a stat name (with format), value format and value */ #define APPEND_NUM_FMT_STAT(name_fmt, num, name, fmt, val) \ klen = snprintf(key_str, STAT_KEY_LEN, name_fmt, num, name); \ vlen = snprintf(val_str, STAT_VAL_LEN, fmt, val); \ add_stats(key_str, klen, val_str, vlen, c); /** Common APPEND_NUM_FMT_STAT format. */ #define APPEND_NUM_STAT(num, name, fmt, val) \ APPEND_NUM_FMT_STAT("%d:%s", num, name, fmt, val) /** Item client flag conversion */ #define FLAGS_CONV(it, flag) { \ if ((it)->it_flags & ITEM_CFLAGS) { \ flag = *((uint32_t *)ITEM_suffix((it))); \ } else { \ flag = 0; \ } \ } #define FLAGS_SIZE(item) (((item)->it_flags & ITEM_CFLAGS) ? sizeof(uint32_t) : 0) /** * Callback for any function producing stats. * * @param key the stat's key * @param klen length of the key * @param val the stat's value in an ascii form (e.g. text form of a number) * @param vlen length of the value * @parm cookie magic callback cookie */ typedef void (*ADD_STAT)(const char *key, const uint16_t klen, const char *val, const uint32_t vlen, const void *cookie); /* * NOTE: If you modify this table you _MUST_ update the function state_text */ /** * Possible states of a connection. */ enum conn_states { conn_listening, /**< the socket which listens for connections */ conn_new_cmd, /**< Prepare connection for next command */ conn_waiting, /**< waiting for a readable socket */ conn_read, /**< reading in a command line */ conn_parse_cmd, /**< try to parse a command from the input buffer */ conn_write, /**< writing out a simple response */ conn_nread, /**< reading in a fixed number of bytes */ conn_swallow, /**< swallowing unnecessary bytes w/o storing */ conn_closing, /**< closing this connection */ conn_mwrite, /**< writing out many items sequentially */ conn_closed, /**< connection is closed */ conn_watch, /**< held by the logger thread as a watcher */ conn_max_state /**< Max state value (used for assertion) */ }; enum bin_substates { bin_no_state, bin_reading_set_header, bin_reading_cas_header, bin_read_set_value, bin_reading_get_key, bin_reading_stat, bin_reading_del_header, bin_reading_incr_header, bin_read_flush_exptime, bin_reading_sasl_auth, bin_reading_sasl_auth_data, bin_reading_touch_key, }; enum protocol { ascii_prot = 3, /* arbitrary value. */ binary_prot, negotiating_prot /* Discovering the protocol */ }; enum network_transport { local_transport, /* Unix sockets*/ tcp_transport, udp_transport }; enum pause_thread_types { PAUSE_WORKER_THREADS = 0, PAUSE_ALL_THREADS, RESUME_ALL_THREADS, RESUME_WORKER_THREADS }; #define IS_TCP(x) (x == tcp_transport) #define IS_UDP(x) (x == udp_transport) #define NREAD_ADD 1 #define NREAD_SET 2 #define NREAD_REPLACE 3 #define NREAD_APPEND 4 #define NREAD_PREPEND 5 #define NREAD_CAS 6 enum store_item_type { NOT_STORED=0, STORED, EXISTS, NOT_FOUND, TOO_LARGE, NO_MEMORY }; enum delta_result_type { OK, NON_NUMERIC, EOM, DELTA_ITEM_NOT_FOUND, DELTA_ITEM_CAS_MISMATCH }; /** Time relative to server start. Smaller than time_t on 64-bit systems. */ // TODO: Move to sub-header. needed in logger.h //typedef unsigned int rel_time_t; /** Use X macros to avoid iterating over the stats fields during reset and * aggregation. No longer have to add new stats in 3+ places. */ #define SLAB_STATS_FIELDS \ X(set_cmds) \ X(get_hits) \ X(touch_hits) \ X(delete_hits) \ X(cas_hits) \ X(cas_badval) \ X(incr_hits) \ X(decr_hits) /** Stats stored per slab (and per thread). */ struct slab_stats { #define X(name) uint64_t name; SLAB_STATS_FIELDS #undef X }; #define THREAD_STATS_FIELDS \ X(get_cmds) \ X(get_misses) \ X(get_expired) \ X(get_flushed) \ X(touch_cmds) \ X(touch_misses) \ X(delete_misses) \ X(incr_misses) \ X(decr_misses) \ X(cas_misses) \ X(meta_cmds) \ X(bytes_read) \ X(bytes_written) \ X(flush_cmds) \ X(conn_yields) /* # of yields for connections (-R option)*/ \ X(auth_cmds) \ X(auth_errors) \ X(idle_kicks) /* idle connections killed */ #ifdef EXTSTORE #define EXTSTORE_THREAD_STATS_FIELDS \ X(get_extstore) \ X(get_aborted_extstore) \ X(get_oom_extstore) \ X(recache_from_extstore) \ X(miss_from_extstore) \ X(badcrc_from_extstore) #endif /** * Stats stored per-thread. */ struct thread_stats { pthread_mutex_t mutex; #define X(name) uint64_t name; THREAD_STATS_FIELDS #ifdef EXTSTORE EXTSTORE_THREAD_STATS_FIELDS #endif #undef X struct slab_stats slab_stats[MAX_NUMBER_OF_SLAB_CLASSES]; uint64_t lru_hits[POWER_LARGEST]; }; /** * Global stats. Only resettable stats should go into this structure. */ struct stats { uint64_t total_items; uint64_t total_conns; uint64_t rejected_conns; uint64_t malloc_fails; uint64_t listen_disabled_num; uint64_t slabs_moved; /* times slabs were moved around */ uint64_t slab_reassign_rescues; /* items rescued during slab move */ uint64_t slab_reassign_evictions_nomem; /* valid items lost during slab move */ uint64_t slab_reassign_inline_reclaim; /* valid items lost during slab move */ uint64_t slab_reassign_chunk_rescues; /* chunked-item chunks recovered */ uint64_t slab_reassign_busy_items; /* valid temporarily unmovable */ uint64_t slab_reassign_busy_deletes; /* refcounted items killed */ uint64_t lru_crawler_starts; /* Number of item crawlers kicked off */ uint64_t lru_maintainer_juggles; /* number of LRU bg pokes */ uint64_t time_in_listen_disabled_us; /* elapsed time in microseconds while server unable to process new connections */ uint64_t log_worker_dropped; /* logs dropped by worker threads */ uint64_t log_worker_written; /* logs written by worker threads */ uint64_t log_watcher_skipped; /* logs watchers missed */ uint64_t log_watcher_sent; /* logs sent to watcher buffers */ #ifdef EXTSTORE uint64_t extstore_compact_lost; /* items lost because they were locked */ uint64_t extstore_compact_rescues; /* items re-written during compaction */ uint64_t extstore_compact_skipped; /* unhit items skipped during compaction */ #endif struct timeval maxconns_entered; /* last time maxconns entered */ }; /** * Global "state" stats. Reflects state that shouldn't be wiped ever. * Ordered for some cache line locality for commonly updated counters. */ struct stats_state { uint64_t curr_items; uint64_t curr_bytes; uint64_t curr_conns; uint64_t hash_bytes; /* size used for hash tables */ unsigned int conn_structs; unsigned int reserved_fds; unsigned int hash_power_level; /* Better hope it's not over 9000 */ bool hash_is_expanding; /* If the hash table is being expanded */ bool accepting_conns; /* whether we are currently accepting */ bool slab_reassign_running; /* slab reassign in progress */ bool lru_crawler_running; /* crawl in progress */ }; #define MAX_VERBOSITY_LEVEL 2 /* When adding a setting, be sure to update process_stat_settings */ /** * Globally accessible settings as derived from the commandline. */ struct settings { size_t maxbytes; int maxconns; int port; int udpport; char *inter; int verbose; rel_time_t oldest_live; /* ignore existing items older than this */ uint64_t oldest_cas; /* ignore existing items with CAS values lower than this */ int evict_to_free; char *socketpath; /* path to unix socket if using local socket */ char *auth_file; /* path to user authentication file */ int access; /* access mask (a la chmod) for unix domain socket */ double factor; /* chunk size growth factor */ int chunk_size; int num_threads; /* number of worker (without dispatcher) libevent threads to run */ int num_threads_per_udp; /* number of worker threads serving each udp socket */ char prefix_delimiter; /* character that marks a key prefix (for stats) */ int detail_enabled; /* nonzero if we're collecting detailed stats */ int reqs_per_event; /* Maximum number of io to process on each io-event. */ bool use_cas; enum protocol binding_protocol; int backlog; int item_size_max; /* Maximum item size */ int slab_chunk_size_max; /* Upper end for chunks within slab pages. */ int slab_page_size; /* Slab's page units. */ volatile sig_atomic_t sig_hup; /* a HUP signal was received but not yet handled */ bool sasl; /* SASL on/off */ bool maxconns_fast; /* Whether or not to early close connections */ bool lru_crawler; /* Whether or not to enable the autocrawler thread */ bool lru_maintainer_thread; /* LRU maintainer background thread */ bool lru_segmented; /* Use split or flat LRU's */ bool slab_reassign; /* Whether or not slab reassignment is allowed */ int slab_automove; /* Whether or not to automatically move slabs */ double slab_automove_ratio; /* youngest must be within pct of oldest */ unsigned int slab_automove_window; /* window mover for algorithm */ int hashpower_init; /* Starting hash power level */ bool shutdown_command; /* allow shutdown command */ int tail_repair_time; /* LRU tail refcount leak repair time */ bool flush_enabled; /* flush_all enabled */ bool dump_enabled; /* whether cachedump/metadump commands work */ char *hash_algorithm; /* Hash algorithm in use */ int lru_crawler_sleep; /* Microsecond sleep between items */ uint32_t lru_crawler_tocrawl; /* Number of items to crawl per run */ int hot_lru_pct; /* percentage of slab space for HOT_LRU */ int warm_lru_pct; /* percentage of slab space for WARM_LRU */ double hot_max_factor; /* HOT tail age relative to COLD tail */ double warm_max_factor; /* WARM tail age relative to COLD tail */ int crawls_persleep; /* Number of LRU crawls to run before sleeping */ bool temp_lru; /* TTL < temporary_ttl uses TEMP_LRU */ uint32_t temporary_ttl; /* temporary LRU threshold */ int idle_timeout; /* Number of seconds to let connections idle */ unsigned int logger_watcher_buf_size; /* size of logger's per-watcher buffer */ unsigned int logger_buf_size; /* size of per-thread logger buffer */ bool drop_privileges; /* Whether or not to drop unnecessary process privileges */ bool watch_enabled; /* allows watch commands to be dropped */ bool relaxed_privileges; /* Relax process restrictions when running testapp */ #ifdef EXTSTORE unsigned int ext_io_threadcount; /* number of IO threads to run. */ unsigned int ext_page_size; /* size in megabytes of storage pages. */ unsigned int ext_item_size; /* minimum size of items to store externally */ unsigned int ext_item_age; /* max age of tail item before storing ext. */ unsigned int ext_low_ttl; /* remaining TTL below this uses own pages */ unsigned int ext_recache_rate; /* counter++ % recache_rate == 0 > recache */ unsigned int ext_wbuf_size; /* read only note for the engine */ unsigned int ext_compact_under; /* when fewer than this many pages, compact */ unsigned int ext_drop_under; /* when fewer than this many pages, drop COLD items */ double ext_max_frag; /* ideal maximum page fragmentation */ double slab_automove_freeratio; /* % of memory to hold free as buffer */ bool ext_drop_unread; /* skip unread items during compaction */ /* per-slab-class free chunk limit */ unsigned int ext_free_memchunks[MAX_NUMBER_OF_SLAB_CLASSES]; #endif #ifdef TLS bool ssl_enabled; /* indicates whether SSL is enabled */ SSL_CTX *ssl_ctx; /* holds the SSL server context which has the server certificate */ char *ssl_chain_cert; /* path to the server SSL chain certificate */ char *ssl_key; /* path to the server key */ int ssl_verify_mode; /* client certificate verify mode */ int ssl_keyformat; /* key format , defult is PEM */ char *ssl_ciphers; /* list of SSL ciphers */ char *ssl_ca_cert; /* certificate with CAs. */ rel_time_t ssl_last_cert_refresh_time; /* time of the last server certificate refresh */ unsigned int ssl_wbuf_size; /* size of the write buffer used by ssl_sendmsg method */ #endif }; extern struct stats stats; extern struct stats_state stats_state; extern time_t process_started; extern struct settings settings; #define ITEM_LINKED 1 #define ITEM_CAS 2 /* temp */ #define ITEM_SLABBED 4 /* Item was fetched at least once in its lifetime */ #define ITEM_FETCHED 8 /* Appended on fetch, removed on LRU shuffling */ #define ITEM_ACTIVE 16 /* If an item's storage are chained chunks. */ #define ITEM_CHUNKED 32 #define ITEM_CHUNK 64 /* ITEM_data bulk is external to item */ #define ITEM_HDR 128 /* additional 4 bytes for item client flags */ #define ITEM_CFLAGS 256 /* item has sent out a token already */ #define ITEM_TOKEN_SENT 512 /* reserved, in case tokens should be a 2-bit count in future */ #define ITEM_TOKEN_RESERVED 1024 /* if item has been marked as a stale value */ #define ITEM_STALE 2048 /** * Structure for storing items within memcached. */ typedef struct _stritem { /* Protected by LRU locks */ struct _stritem *next; struct _stritem *prev; /* Rest are protected by an item lock */ struct _stritem *h_next; /* hash chain next */ rel_time_t time; /* least recent access */ rel_time_t exptime; /* expire time */ int nbytes; /* size of data */ unsigned short refcount; uint16_t it_flags; /* ITEM_* above */ uint8_t slabs_clsid;/* which slab class we're in */ uint8_t nkey; /* key length, w/terminating null and padding */ /* this odd type prevents type-punning issues when we do * the little shuffle to save space when not using CAS. */ union { uint64_t cas; char end; } data[]; /* if it_flags & ITEM_CAS we have 8 bytes CAS */ /* then null-terminated key */ /* then " flags length\r\n" (no terminating null) */ /* then data with terminating \r\n (no terminating null; it's binary!) */ } item; // TODO: If we eventually want user loaded modules, we can't use an enum :( enum crawler_run_type { CRAWLER_AUTOEXPIRE=0, CRAWLER_EXPIRED, CRAWLER_METADUMP }; typedef struct { struct _stritem *next; struct _stritem *prev; struct _stritem *h_next; /* hash chain next */ rel_time_t time; /* least recent access */ rel_time_t exptime; /* expire time */ int nbytes; /* size of data */ unsigned short refcount; uint16_t it_flags; /* ITEM_* above */ uint8_t slabs_clsid;/* which slab class we're in */ uint8_t nkey; /* key length, w/terminating null and padding */ uint32_t remaining; /* Max keys to crawl per slab per invocation */ uint64_t reclaimed; /* items reclaimed during this crawl. */ uint64_t unfetched; /* items reclaimed unfetched during this crawl. */ uint64_t checked; /* items examined during this crawl. */ } crawler; /* Header when an item is actually a chunk of another item. */ typedef struct _strchunk { struct _strchunk *next; /* points within its own chain. */ struct _strchunk *prev; /* can potentially point to the head. */ struct _stritem *head; /* always points to the owner chunk */ int size; /* available chunk space in bytes */ int used; /* chunk space used */ int nbytes; /* used. */ unsigned short refcount; /* used? */ uint16_t it_flags; /* ITEM_* above. */ uint8_t slabs_clsid; /* Same as above. */ uint8_t orig_clsid; /* For obj hdr chunks slabs_clsid is fake. */ char data[]; } item_chunk; #ifdef NEED_ALIGN static inline char *ITEM_schunk(item *it) { int offset = it->nkey + 1 + ((it->it_flags & ITEM_CFLAGS) ? sizeof(uint32_t) : 0) + ((it->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0); int remain = offset % 8; if (remain != 0) { offset += 8 - remain; } return ((char *) &(it->data)) + offset; } #else #define ITEM_schunk(item) ((char*) &((item)->data) + (item)->nkey + 1 \ + (((item)->it_flags & ITEM_CFLAGS) ? sizeof(uint32_t) : 0) \ + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0)) #endif #ifdef EXTSTORE typedef struct { unsigned int page_version; /* from IO header */ unsigned int offset; /* from IO header */ unsigned short page_id; /* from IO header */ } item_hdr; #endif typedef struct { pthread_t thread_id; /* unique ID of this thread */ struct event_base *base; /* libevent handle this thread uses */ struct event notify_event; /* listen event for notify pipe */ int notify_receive_fd; /* receiving end of notify pipe */ int notify_send_fd; /* sending end of notify pipe */ struct thread_stats stats; /* Stats generated by this thread */ struct conn_queue *new_conn_queue; /* queue of new connections to handle */ cache_t *suffix_cache; /* suffix cache */ #ifdef EXTSTORE cache_t *io_cache; /* IO objects */ void *storage; /* data object for storage system */ #endif logger *l; /* logger buffer */ void *lru_bump_buf; /* async LRU bump buffer */ #ifdef TLS char *ssl_wbuf; #endif } LIBEVENT_THREAD; typedef struct conn conn; #ifdef EXTSTORE typedef struct _io_wrap { obj_io io; struct _io_wrap *next; conn *c; item *hdr_it; /* original header item. */ unsigned int iovec_start; /* start of the iovecs for this IO */ unsigned int iovec_count; /* total number of iovecs */ unsigned int iovec_data; /* specific index of data iovec */ bool miss; /* signal a miss to unlink hdr_it */ bool badcrc; /* signal a crc failure */ bool active; /* tells if IO was dispatched or not */ } io_wrap; #endif /** * The structure representing a connection into memcached. */ struct conn { sasl_conn_t *sasl_conn; int sfd; bool sasl_started; bool authenticated; bool set_stale; bool mset_res; /** uses mset format for return code */ #ifdef TLS SSL *ssl; char *ssl_wbuf; bool ssl_enabled; #endif enum conn_states state; enum bin_substates substate; rel_time_t last_cmd_time; struct event event; short ev_flags; short which; /** which events were just triggered */ char *rbuf; /** buffer to read commands into */ char *rcurr; /** but if we parsed some already, this is where we stopped */ int rsize; /** total allocated size of rbuf */ int rbytes; /** how much data, starting from rcur, do we have unparsed */ char *wbuf; char *wcurr; int wsize; int wbytes; /** which state to go into after finishing current write */ enum conn_states write_and_go; void *write_and_free; /** free this memory after finishing writing */ char *ritem; /** when we read in an item's value, it goes here */ int rlbytes; /* data for the nread state */ /** * item is used to hold an item structure created after reading the command * line of set/add/replace commands, but before we finished reading the actual * data. The data is read into ITEM_data(item) to avoid extra copying. */ void *item; /* for commands set/add/replace */ /* data for the swallow state */ int sbytes; /* how many bytes to swallow */ /* data for the mwrite state */ struct iovec *iov; int iovsize; /* number of elements allocated in iov[] */ int iovused; /* number of elements used in iov[] */ struct msghdr *msglist; int msgsize; /* number of elements allocated in msglist[] */ int msgused; /* number of elements used in msglist[] */ int msgcurr; /* element in msglist[] being transmitted now */ int msgbytes; /* number of bytes in current msg */ item **ilist; /* list of items to write out */ int isize; item **icurr; int ileft; char **suffixlist; int suffixsize; char **suffixcurr; int suffixleft; #ifdef EXTSTORE int io_wrapleft; unsigned int recache_counter; io_wrap *io_wraplist; /* linked list of io_wraps */ bool io_queued; /* FIXME: debugging flag */ #endif enum protocol protocol; /* which protocol this connection speaks */ enum network_transport transport; /* what transport is used by this connection */ /* data for UDP clients */ int request_id; /* Incoming UDP request ID, if this is a UDP "connection" */ struct sockaddr_in6 request_addr; /* udp: Who sent the most recent request */ socklen_t request_addr_size; unsigned char *hdrbuf; /* udp packet headers */ int hdrsize; /* number of headers' worth of space is allocated */ bool noreply; /* True if the reply should not be sent. */ /* current stats command */ struct { char *buffer; size_t size; size_t offset; } stats; /* Binary protocol stuff */ /* This is where the binary header goes */ protocol_binary_request_header binary_header; uint64_t cas; /* the cas to return */ short cmd; /* current command being processed */ int opaque; int keylen; conn *next; /* Used for generating a list of conn structures */ LIBEVENT_THREAD *thread; /* Pointer to the thread object serving this connection */ int (*try_read_command)(conn *c); /* pointer for top level input parser */ ssize_t (*read)(conn *c, void *buf, size_t count); ssize_t (*sendmsg)(conn *c, struct msghdr *msg, int flags); ssize_t (*write)(conn *c, void *buf, size_t count); }; /* array of conn structures, indexed by file descriptor */ extern conn **conns; /* current time of day (updated periodically) */ extern volatile rel_time_t current_time; /* TODO: Move to slabs.h? */ extern volatile int slab_rebalance_signal; struct slab_rebalance { void *slab_start; void *slab_end; void *slab_pos; int s_clsid; int d_clsid; uint32_t busy_items; uint32_t rescues; uint32_t evictions_nomem; uint32_t inline_reclaim; uint32_t chunk_rescues; uint32_t busy_deletes; uint32_t busy_loops; uint8_t done; uint8_t *completed; }; extern struct slab_rebalance slab_rebal; #ifdef EXTSTORE extern void *ext_storage; #endif /* * Functions */ void do_accept_new_conns(const bool do_accept); enum delta_result_type do_add_delta(conn *c, const char *key, const size_t nkey, const bool incr, const int64_t delta, char *buf, uint64_t *cas, const uint32_t hv); enum store_item_type do_store_item(item *item, int comm, conn* c, const uint32_t hv); conn *conn_new(const int sfd, const enum conn_states init_state, const int event_flags, const int read_buffer_size, enum network_transport transport, struct event_base *base, void *ssl); void conn_worker_readd(conn *c); extern int daemonize(int nochdir, int noclose); #define mutex_lock(x) pthread_mutex_lock(x) #define mutex_unlock(x) pthread_mutex_unlock(x) #include "stats_prefix.h" #include "slabs.h" #include "assoc.h" #include "items.h" #include "crawler.h" #include "trace.h" #include "hash.h" #include "util.h" /* * Functions such as the libevent-related calls that need to do cross-thread * communication in multithreaded mode (rather than actually doing the work * in the current thread) are called via "dispatch_" frontends, which are * also #define-d to directly call the underlying code in singlethreaded mode. */ void memcached_thread_init(int nthreads, void *arg); void redispatch_conn(conn *c); void dispatch_conn_new(int sfd, enum conn_states init_state, int event_flags, int read_buffer_size, enum network_transport transport, void *ssl); void sidethread_conn_close(conn *c); /* Lock wrappers for cache functions that are called from main loop. */ enum delta_result_type add_delta(conn *c, const char *key, const size_t nkey, bool incr, const int64_t delta, char *buf, uint64_t *cas); void accept_new_conns(const bool do_accept); conn *conn_from_freelist(void); bool conn_add_to_freelist(conn *c); void conn_close_idle(conn *c); item *item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes); #define DO_UPDATE true #define DONT_UPDATE false item *item_get(const char *key, const size_t nkey, conn *c, const bool do_update); item *item_get_locked(const char *key, const size_t nkey, conn *c, const bool do_update, uint32_t *hv); item *item_touch(const char *key, const size_t nkey, uint32_t exptime, conn *c); int item_link(item *it); void item_remove(item *it); int item_replace(item *it, item *new_it, const uint32_t hv); void item_unlink(item *it); void item_lock(uint32_t hv); void *item_trylock(uint32_t hv); void item_trylock_unlock(void *arg); void item_unlock(uint32_t hv); void pause_threads(enum pause_thread_types type); void stop_threads(void); int stop_conn_timeout_thread(void); #define refcount_incr(it) ++(it->refcount) #define refcount_decr(it) --(it->refcount) void STATS_LOCK(void); void STATS_UNLOCK(void); void threadlocal_stats_reset(void); void threadlocal_stats_aggregate(struct thread_stats *stats); void slab_stats_aggregate(struct thread_stats *stats, struct slab_stats *out); /* Stat processing functions */ void append_stat(const char *name, ADD_STAT add_stats, conn *c, const char *fmt, ...); enum store_item_type store_item(item *item, int comm, conn *c); #if HAVE_DROP_PRIVILEGES extern void setup_privilege_violations_handler(void); extern void drop_privileges(void); #else #define setup_privilege_violations_handler() #define drop_privileges() #endif #if HAVE_DROP_WORKER_PRIVILEGES extern void drop_worker_privileges(void); #else #define drop_worker_privileges() #endif /* If supported, give compiler hints for branch prediction. */ #if !defined(__GNUC__) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) #endif #define likely(x) __builtin_expect((x),1) #define unlikely(x) __builtin_expect((x),0) memcached-1.5.22/compile0000755000175000017500000001624513256032133012014 00000000000000#! /bin/sh # Wrapper for compilers which do not understand '-c -o'. scriptversion=2012-10-14.11; # UTC # Copyright (C) 1999-2014 Free Software Foundation, Inc. # Written by Tom Tromey . # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. # This file is maintained in Automake, please report # bugs to or send patches to # . nl=' ' # We need space, tab and new line, in precisely that order. Quoting is # there to prevent tools from complaining about whitespace usage. IFS=" "" $nl" file_conv= # func_file_conv build_file lazy # Convert a $build file to $host form and store it in $file # Currently only supports Windows hosts. If the determined conversion # type is listed in (the comma separated) LAZY, no conversion will # take place. func_file_conv () { file=$1 case $file in / | /[!/]*) # absolute file, and not a UNC file if test -z "$file_conv"; then # lazily determine how to convert abs files case `uname -s` in MINGW*) file_conv=mingw ;; CYGWIN*) file_conv=cygwin ;; *) file_conv=wine ;; esac fi case $file_conv/,$2, in *,$file_conv,*) ;; mingw/*) file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'` ;; cygwin/*) file=`cygpath -m "$file" || echo "$file"` ;; wine/*) file=`winepath -w "$file" || echo "$file"` ;; esac ;; esac } # func_cl_dashL linkdir # Make cl look for libraries in LINKDIR func_cl_dashL () { func_file_conv "$1" if test -z "$lib_path"; then lib_path=$file else lib_path="$lib_path;$file" fi linker_opts="$linker_opts -LIBPATH:$file" } # func_cl_dashl library # Do a library search-path lookup for cl func_cl_dashl () { lib=$1 found=no save_IFS=$IFS IFS=';' for dir in $lib_path $LIB do IFS=$save_IFS if $shared && test -f "$dir/$lib.dll.lib"; then found=yes lib=$dir/$lib.dll.lib break fi if test -f "$dir/$lib.lib"; then found=yes lib=$dir/$lib.lib break fi if test -f "$dir/lib$lib.a"; then found=yes lib=$dir/lib$lib.a break fi done IFS=$save_IFS if test "$found" != yes; then lib=$lib.lib fi } # func_cl_wrapper cl arg... # Adjust compile command to suit cl func_cl_wrapper () { # Assume a capable shell lib_path= shared=: linker_opts= for arg do if test -n "$eat"; then eat= else case $1 in -o) # configure might choose to run compile as 'compile cc -o foo foo.c'. eat=1 case $2 in *.o | *.[oO][bB][jJ]) func_file_conv "$2" set x "$@" -Fo"$file" shift ;; *) func_file_conv "$2" set x "$@" -Fe"$file" shift ;; esac ;; -I) eat=1 func_file_conv "$2" mingw set x "$@" -I"$file" shift ;; -I*) func_file_conv "${1#-I}" mingw set x "$@" -I"$file" shift ;; -l) eat=1 func_cl_dashl "$2" set x "$@" "$lib" shift ;; -l*) func_cl_dashl "${1#-l}" set x "$@" "$lib" shift ;; -L) eat=1 func_cl_dashL "$2" ;; -L*) func_cl_dashL "${1#-L}" ;; -static) shared=false ;; -Wl,*) arg=${1#-Wl,} save_ifs="$IFS"; IFS=',' for flag in $arg; do IFS="$save_ifs" linker_opts="$linker_opts $flag" done IFS="$save_ifs" ;; -Xlinker) eat=1 linker_opts="$linker_opts $2" ;; -*) set x "$@" "$1" shift ;; *.cc | *.CC | *.cxx | *.CXX | *.[cC]++) func_file_conv "$1" set x "$@" -Tp"$file" shift ;; *.c | *.cpp | *.CPP | *.lib | *.LIB | *.Lib | *.OBJ | *.obj | *.[oO]) func_file_conv "$1" mingw set x "$@" "$file" shift ;; *) set x "$@" "$1" shift ;; esac fi shift done if test -n "$linker_opts"; then linker_opts="-link$linker_opts" fi exec "$@" $linker_opts exit 1 } eat= case $1 in '') echo "$0: No command. Try '$0 --help' for more information." 1>&2 exit 1; ;; -h | --h*) cat <<\EOF Usage: compile [--help] [--version] PROGRAM [ARGS] Wrapper for compilers which do not understand '-c -o'. Remove '-o dest.o' from ARGS, run PROGRAM with the remaining arguments, and rename the output as expected. If you are trying to build a whole package this is not the right script to run: please start by reading the file 'INSTALL'. Report bugs to . EOF exit $? ;; -v | --v*) echo "compile $scriptversion" exit $? ;; cl | *[/\\]cl | cl.exe | *[/\\]cl.exe ) func_cl_wrapper "$@" # Doesn't return... ;; esac ofile= cfile= for arg do if test -n "$eat"; then eat= else case $1 in -o) # configure might choose to run compile as 'compile cc -o foo foo.c'. # So we strip '-o arg' only if arg is an object. eat=1 case $2 in *.o | *.obj) ofile=$2 ;; *) set x "$@" -o "$2" shift ;; esac ;; *.c) cfile=$1 set x "$@" "$1" shift ;; *) set x "$@" "$1" shift ;; esac fi shift done if test -z "$ofile" || test -z "$cfile"; then # If no '-o' option was seen then we might have been invoked from a # pattern rule where we don't need one. That is ok -- this is a # normal compilation that the losing compiler can handle. If no # '.c' file was seen then we are probably linking. That is also # ok. exec "$@" fi # Name of file we expect compiler to create. cofile=`echo "$cfile" | sed 's|^.*[\\/]||; s|^[a-zA-Z]:||; s/\.c$/.o/'` # Create the lock directory. # Note: use '[/\\:.-]' here to ensure that we don't use the same name # that we are using for the .o file. Also, base the name on the expected # object file name, since that is what matters with a parallel build. lockdir=`echo "$cofile" | sed -e 's|[/\\:.-]|_|g'`.d while true; do if mkdir "$lockdir" >/dev/null 2>&1; then break fi sleep 1 done # FIXME: race condition here if user kills between mkdir and trap. trap "rmdir '$lockdir'; exit 1" 1 2 15 # Run the compile. "$@" ret=$? if test -f "$cofile"; then test "$cofile" = "$ofile" || mv "$cofile" "$ofile" elif test -f "${cofile}bj"; then test "${cofile}bj" = "$ofile" || mv "${cofile}bj" "$ofile" fi rmdir "$lockdir" exit $ret # Local Variables: # mode: shell-script # sh-indentation: 2 # eval: (add-hook 'write-file-hooks 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" # time-stamp-time-zone: "UTC" # time-stamp-end: "; # UTC" # End: memcached-1.5.22/missing0000755000175000017500000001533013256032133012027 00000000000000#! /bin/sh # Common wrapper for a few potentially missing GNU programs. scriptversion=2013-10-28.13; # UTC # Copyright (C) 1996-2014 Free Software Foundation, Inc. # Originally written by Fran,cois Pinard , 1996. # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2, or (at your option) # any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. if test $# -eq 0; then echo 1>&2 "Try '$0 --help' for more information" exit 1 fi case $1 in --is-lightweight) # Used by our autoconf macros to check whether the available missing # script is modern enough. exit 0 ;; --run) # Back-compat with the calling convention used by older automake. shift ;; -h|--h|--he|--hel|--help) echo "\ $0 [OPTION]... PROGRAM [ARGUMENT]... Run 'PROGRAM [ARGUMENT]...', returning a proper advice when this fails due to PROGRAM being missing or too old. Options: -h, --help display this help and exit -v, --version output version information and exit Supported PROGRAM values: aclocal autoconf autoheader autom4te automake makeinfo bison yacc flex lex help2man Version suffixes to PROGRAM as well as the prefixes 'gnu-', 'gnu', and 'g' are ignored when checking the name. Send bug reports to ." exit $? ;; -v|--v|--ve|--ver|--vers|--versi|--versio|--version) echo "missing $scriptversion (GNU Automake)" exit $? ;; -*) echo 1>&2 "$0: unknown '$1' option" echo 1>&2 "Try '$0 --help' for more information" exit 1 ;; esac # Run the given program, remember its exit status. "$@"; st=$? # If it succeeded, we are done. test $st -eq 0 && exit 0 # Also exit now if we it failed (or wasn't found), and '--version' was # passed; such an option is passed most likely to detect whether the # program is present and works. case $2 in --version|--help) exit $st;; esac # Exit code 63 means version mismatch. This often happens when the user # tries to use an ancient version of a tool on a file that requires a # minimum version. if test $st -eq 63; then msg="probably too old" elif test $st -eq 127; then # Program was missing. msg="missing on your system" else # Program was found and executed, but failed. Give up. exit $st fi perl_URL=http://www.perl.org/ flex_URL=http://flex.sourceforge.net/ gnu_software_URL=http://www.gnu.org/software program_details () { case $1 in aclocal|automake) echo "The '$1' program is part of the GNU Automake package:" echo "<$gnu_software_URL/automake>" echo "It also requires GNU Autoconf, GNU m4 and Perl in order to run:" echo "<$gnu_software_URL/autoconf>" echo "<$gnu_software_URL/m4/>" echo "<$perl_URL>" ;; autoconf|autom4te|autoheader) echo "The '$1' program is part of the GNU Autoconf package:" echo "<$gnu_software_URL/autoconf/>" echo "It also requires GNU m4 and Perl in order to run:" echo "<$gnu_software_URL/m4/>" echo "<$perl_URL>" ;; esac } give_advice () { # Normalize program name to check for. normalized_program=`echo "$1" | sed ' s/^gnu-//; t s/^gnu//; t s/^g//; t'` printf '%s\n' "'$1' is $msg." configure_deps="'configure.ac' or m4 files included by 'configure.ac'" case $normalized_program in autoconf*) echo "You should only need it if you modified 'configure.ac'," echo "or m4 files included by it." program_details 'autoconf' ;; autoheader*) echo "You should only need it if you modified 'acconfig.h' or" echo "$configure_deps." program_details 'autoheader' ;; automake*) echo "You should only need it if you modified 'Makefile.am' or" echo "$configure_deps." program_details 'automake' ;; aclocal*) echo "You should only need it if you modified 'acinclude.m4' or" echo "$configure_deps." program_details 'aclocal' ;; autom4te*) echo "You might have modified some maintainer files that require" echo "the 'autom4te' program to be rebuilt." program_details 'autom4te' ;; bison*|yacc*) echo "You should only need it if you modified a '.y' file." echo "You may want to install the GNU Bison package:" echo "<$gnu_software_URL/bison/>" ;; lex*|flex*) echo "You should only need it if you modified a '.l' file." echo "You may want to install the Fast Lexical Analyzer package:" echo "<$flex_URL>" ;; help2man*) echo "You should only need it if you modified a dependency" \ "of a man page." echo "You may want to install the GNU Help2man package:" echo "<$gnu_software_URL/help2man/>" ;; makeinfo*) echo "You should only need it if you modified a '.texi' file, or" echo "any other file indirectly affecting the aspect of the manual." echo "You might want to install the Texinfo package:" echo "<$gnu_software_URL/texinfo/>" echo "The spurious makeinfo call might also be the consequence of" echo "using a buggy 'make' (AIX, DU, IRIX), in which case you might" echo "want to install GNU make:" echo "<$gnu_software_URL/make/>" ;; *) echo "You might have modified some files without having the proper" echo "tools for further handling them. Check the 'README' file, it" echo "often tells you about the needed prerequisites for installing" echo "this package. You may also peek at any GNU archive site, in" echo "case some other package contains this missing '$1' program." ;; esac } give_advice "$1" | sed -e '1s/^/WARNING: /' \ -e '2,$s/^/ /' >&2 # Propagate the correct exit status (expected to be 127 for a program # not found, 63 for a program that failed due to version mismatch). exit $st # Local variables: # eval: (add-hook 'write-file-hooks 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" # time-stamp-time-zone: "UTC" # time-stamp-end: "; # UTC" # End: memcached-1.5.22/scripts/0000755000175000017500000000000013611703161012176 500000000000000memcached-1.5.22/scripts/start-memcached0000775000175000017500000000532213461170555015121 00000000000000#!/usr/bin/perl -w # start-memcached # 2003/2004 - Jay Bonci # This script handles the parsing of the /etc/memcached.conf file # and was originally created for the Debian distribution. # Anyone may use this little script under the same terms as # memcached itself. use POSIX qw(setsid); use strict; if($> != 0 and $< != 0) { print STDERR "Only root wants to run start-memcached.\n"; exit; } my $params; my $etchandle; my $etcfile = "/etc/memcached.conf"; # This script assumes that memcached is located at /usr/bin/memcached, and # that the pidfile is writable at /var/run/memcached.pid my $memcached = "/usr/bin/memcached"; my $pidfile = "/var/run/memcached.pid"; if (scalar(@ARGV) == 2) { $etcfile = shift(@ARGV); $pidfile = shift(@ARGV); } # If we don't get a valid logfile parameter in the /etc/memcached.conf file, # we'll just throw away all of our in-daemon output. We need to re-tie it so # that non-bash shells will not hang on logout. Thanks to Michael Renner for # the tip my $fd_reopened = "/dev/null"; sub handle_logfile { my ($logfile) = @_; $fd_reopened = $logfile; } sub reopen_logfile { my ($logfile) = @_; open *STDERR, ">>$logfile"; open *STDOUT, ">>$logfile"; open *STDIN, ">>/dev/null"; $fd_reopened = $logfile; } # This is set up in place here to support other non -[a-z] directives my $conf_directives = { "logfile" => \&handle_logfile, }; if(open $etchandle, $etcfile) { foreach my $line (<$etchandle>) { $line ||= ""; $line =~ s/(?{$directive}->($arg); next; } push @$params, $line; } }else{ $params = []; } push @$params, "-u root" unless(grep "-u", @$params); $params = join " ", @$params; if(-e $pidfile) { open PIDHANDLE, "$pidfile"; my $localpid = ; close PIDHANDLE; chomp $localpid; if(-d "/proc/$localpid") { print STDERR "memcached is already running.\n"; exit; }else{ unlink $pidfile; } } my $pid = fork(); if($pid == 0) { # setsid makes us the session leader setsid(); reopen_logfile($fd_reopened); # must fork again now that tty is closed $pid = fork(); if ($pid) { if(open PIDHANDLE,">$pidfile") { print PIDHANDLE $pid; close PIDHANDLE; }else{ print STDERR "Can't write pidfile to $pidfile.\n"; } exit(0); } exec "$memcached $params"; exit(0); } memcached-1.5.22/scripts/memcached-automove-extstore0000775000175000017500000002753213461170555017505 00000000000000#!/usr/bin/python3 # Copyright 2017 Facebook. # Licensed under the same terms as memcached itself. import argparse import socket import sys import re import traceback from time import sleep, time parser = argparse.ArgumentParser(description="daemon for rebalancing slabs") parser.add_argument("--host", help="host to connect to", default="localhost:11211", metavar="HOST:PORT") parser.add_argument("-s", "--sleep", help="seconds between runs", type=int, default="1") parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("-a", "--automove", action="store_true", default=False, help="enable automatic page rebalancing") parser.add_argument("-w", "--window", type=int, default="30", help="rolling window size for decision history") parser.add_argument("-r", "--ratio", type=float, default=0.8, help="ratio limiting distance between low/high class ages") parser.add_argument("-f", "--free", type=float, default=0.005, help="free chunks/pages buffer ratio") parser.add_argument("-z", "--size", type=int, default=512, help="item size cutoff for storage") args = parser.parse_args() host, port = args.host.split(':') MIN_PAGES_FOR_SOURCE = 2 MIN_PAGES_FOR_RECLAIM = 2.5 MIN_PAGES_FREE = 1.5 MEMCHECK_PERIOD = 60 def window_check(history, sid, key): total = 0 for window in history['w']: s = window.get(sid) if s and s.get(key): total += s.get(key) return total def window_key_check(history, key): total = 0 for window in history['w']: v = window.get(key) if v: total += v return total def determine_move(history, stats, diffs, memfree): """ Figure out of a page move is in order. - Use as much memory as possible to hold items, reducing the load on flash. - tries to keep the global free page pool inbetween poolmin/poolmax. - avoids flapping as much as possible: - only pull pages off of a class if it hasn't recently evicted or allocated pages. - only pull pages off if a sufficient number of free chunks are available. - if global pool is below minimum remove pages from oldest large class. - if global pool is above maximum, move pages to youngest large class. - extstore manages a desired number of free chunks in each slab class. - automover adjusts above limits once per minute based on current sizes. - if youngest is below the age ratio limit of oldest, move a page to it. """ # rotate windows history['w'].append({}) if (len(history['w']) > args.window): history['w'].pop(0) w = history['w'][-1] oldest = (-1, 0) youngest = (-1, sys.maxsize) too_free = False # Most bytes free decision = (-1, -1) if int(stats['slab_global_page_pool']) < memfree[0] / 2: w['slab_pool_low'] = 1 if int(stats['slab_global_page_pool']) > memfree[0]: w['slab_pool_high'] = 1 if args.verbose: print("global pool: [{}]".format(stats['slab_global_page_pool'])) pool_low = window_key_check(history, 'slab_pool_low') for sid, slab in diffs.items(): small_slab = False free_enough = False # Only balance larger slab classes if slab['chunk_size'] < args.size: small_slab = True w[sid] = {} if 'evicted_d' not in slab or 'total_pages_d' not in slab: continue # mark this window as dirty if total pages increases or evictions # happened if slab['total_pages_d'] > 0: w[sid]['dirty'] = 1 if slab['evicted_d'] > 0: w[sid]['dirty'] = 1 w[sid]['ev'] = 1 if slab['free_chunks'] > memfree[sid]: free_enough = True if memfree[sid] > 0 and slab['free_chunks'] > (memfree[sid] * 2): w[sid]['excess_free'] = 1 w[sid]['age'] = slab['age'] age = window_check(history, sid, 'age') / len(history['w']) # if > 2.5 pages free, and not dirty, reassign to global page pool if slab['free_chunks'] > slab['chunks_per_page'] * MIN_PAGES_FOR_RECLAIM and too_free == False: dirt = window_check(history, sid, 'dirty') excess = window_check(history, sid, 'excess_free') if small_slab == True and dirt == 0: # If we're a small slab, don't hang on to free memory forever. decision = (sid, 0) too_free = True elif small_slab == False and dirt == 0 \ and excess >= len(history['w']): decision = (sid, 0) too_free = True # are we the oldest slab class? (and a valid target) # don't consider for young if we've recently given it unused memory if small_slab == False: if age > oldest[1] and slab['total_pages'] > MIN_PAGES_FOR_SOURCE: oldest = (sid, age) if age < youngest[1] and slab['total_pages'] > 0 \ and window_check(history, sid, 'excess_free') < len(history['w']) \ and not (window_check(history, sid, 'relaxed') and free_enough): youngest = (sid, age) if w.get('slab_pool_high') and youngest[0] != -1: # if global pool is too high, feed youngest large class. if slab['free_chunks'] <= memfree[youngest[0]]: decision = (0, youngest[0]) w[youngest[0]]['relaxed'] = 1 elif too_free == False and pool_low and oldest[0] != -1: # if pool is too low, take from oldest large class. if args.verbose: print("oldest: [class: {}] [age: {:.2f}]".format(int(oldest[0]), oldest[1])) decision = (oldest[0], 0) elif too_free == False and youngest[0] != -1 and oldest[0] != -1 and youngest[0] != oldest[0]: # youngest is outside of the tolerance ratio, move a page around. if args.verbose: print("old: [class: {}] [age: {:.2f}]\nyoung: [class: {}] [age: {:.2f}]".format( int(oldest[0]), oldest[1], int(youngest[0]), youngest[1])) slab = diffs[youngest[0]] #print("F:{} L:{} Y:{} R:{}".format(slab['free_chunks'], memfree[youngest[0]], int(youngest[1]), int(oldest[1] * args.ratio))) if youngest[1] < oldest[1] * args.ratio: w[youngest[0]]['relaxed'] = 1 if slab['free_chunks'] <= memfree[youngest[0]]: decision = (0, youngest[0]) if (len(history['w']) >= args.window): return decision return (-1, -1) def run_move(s, decision): s.write("slabs reassign " + str(decision[0]) + " " + str(decision[1]) + "\r\n") line = s.readline().rstrip() if args.verbose: print("move result:", line) def diff_stats(before, after): """ fills out "diffs" as deltas between before/after, and "totals" as the sum of all slab classes. "_d" postfix to keys means the delta between before/after. non-postfix keys are total as of 'after's reading. """ diffs = {} totals = {} for slabid in after.keys(): sb = before.get(slabid) sa = after.get(slabid) if not (sb and sa): continue slab = sa.copy() for k in sa.keys(): if k not in sb: continue if k not in totals: totals[k] = 0 totals[k + '_d'] = 0 if k + '_d' not in slab: slab[k + '_d'] = 0 if re.search(r"^\d+$", sa[k]): totals[k] += int(sa[k]) slab[k] = int(sa[k]) slab[k + '_d'] = int(sa[k]) - int(sb[k]) totals[k + '_d'] += int(sa[k]) - int(sb[k]) slab['slab'] = slabid diffs[slabid] = slab return (diffs, totals) def read_slab_stats(s): slabs = {} for statcmd in ['items', 'slabs']: #print("stat cmd: " + statcmd) # FIXME: Formatting s.write("stats " + statcmd + "\r\n") while True: line = s.readline().rstrip() if line.startswith("END"): break m = re.match(r"^STAT (?:items:)?(\d+):(\S+) (\S+)", line) if m: (slab, var, val) = m.groups() if slab not in slabs: slabs[slab] = {} slabs[slab][var] = val #print("line: " + line) return slabs # HACK: lets look at 'evictions' being nonzero to indicate memory filled at some point. def read_stats(s): stats = {} s.write("stats\r\n") while True: line = s.readline().rstrip() if line.startswith("END"): break m = re.match(r"^STAT (\S+) (\S+)", line) if m: (key, val) = m.groups() stats[key] = val return stats def pct(num, divisor): if not divisor: return 0 return (num / divisor) def show_detail(diffs, totals): """ just a pretty printer for some extra data """ print("\n {:2s}: {:8s} (pct ) {:10s} (pct ) {:6s} (pct) {:6s}".format('sb', 'evicted', 'items', 'pages', 'age')) for sid, slab in diffs.items(): if 'evicted_d' not in slab: continue print(" {:2d}: {:8d} ({:.2f}%) {:10d} ({:.4f}%) {:6d} ({:.2f}%) {:6d}".format( int(sid), slab['evicted_d'], pct(slab['evicted_d'], totals['evicted_d']), slab['number'], pct(slab['number'], totals['number']), slab['total_pages'], pct(slab['total_pages'], totals['total_pages']), slab['age'])) def memfree_check(s, diffs, totals): info = {} # manage about this many free chunks in each slab class. for sid, slab in diffs.items(): if sid == 0: continue hold_free = int((slab['used_chunks'] + slab['free_chunks']) * args.free) # Hold a minimum of 1.5 pages so page moves are unlikely to lose items. if slab['chunks_per_page'] * MIN_PAGES_FREE > hold_free: hold_free = int(slab['chunks_per_page'] * MIN_PAGES_FREE) info[sid] = hold_free # TODO: only adjust if different? s.write("extstore free_memchunks {} {}\r\n".format(sid, hold_free)) s.readline() # how many pages to leave in the global pool. info[0] = int(totals['total_pages'] * args.free) return info stats_pre = {} history = { 'w': [{}] } memfree = { 0: 2 } last_memfree_check = 0 while True: try: with socket.create_connection((host, port), 5) as c: s = c.makefile(mode="rw", buffering=1) s.write("slabs automove 0\r\n") print(s.readline().rstrip()) while True: stats_post = read_slab_stats(s) stats = read_stats(s) (diffs, totals) = diff_stats(stats_pre, stats_post) #if args.verbose: # show_detail(diffs, totals) if int(stats['evictions']) > 0: if (last_memfree_check < time() - 60) and totals.get('total_pages'): memfree = memfree_check(s, diffs, totals) last_memfree_check = time() decision = (-1, -1) decision = determine_move(history, stats, diffs, memfree) if int(decision[0]) > 0 and int(decision[1]) >= 0: print("moving page from, to:", decision) if args.automove: run_move(s, decision) # Minimize sleeping if we just moved a page to global pool. # Improves responsiveness during flushes/quick changes. if decision[1] == 0: continue else: sleep(args.sleep) stats_pre = stats_post except: err = sys.exc_info() print("disconnected:", err[0], err[1]) traceback.print_exc() stats_pre = {} history = { 'w': [{}] } sleep(args.sleep) memcached-1.5.22/scripts/memcached-tool0000775000175000017500000001406413611703161014734 00000000000000#!/usr/bin/perl # # memcached-tool: # stats/management tool for memcached. # # Author: # Brad Fitzpatrick # # Contributor: # Andrey Niakhaichyk # # License: # public domain. I give up all rights to this # tool. modify and copy at will. # use strict; use IO::Socket::INET; my $addr = shift; my $mode = shift || "display"; my ($from, $to); my $limit; if ($mode eq "display") { undef $mode if @ARGV; } elsif ($mode eq "move") { $from = shift; $to = shift; undef $mode if $from < 6 || $from > 17; undef $mode if $to < 6 || $to > 17; print STDERR "ERROR: parameters out of range\n\n" unless $mode; } elsif ($mode eq 'dump') { if (@ARGV) { $limit = shift; undef $mode if $limit < 1; print STDERR "ERROR: invalid limit (should be a positive number)\n\n" unless $mode; } } elsif ($mode eq 'stats') { ; } elsif ($mode eq 'settings') { ; } elsif ($mode eq 'sizes') { ; } else { undef $mode; } undef $mode if @ARGV; die "Usage: memcached-tool [mode]\n memcached-tool 10.0.0.5:11211 display # shows slabs memcached-tool 10.0.0.5:11211 # same. (default is display) memcached-tool 10.0.0.5:11211 stats # shows general stats memcached-tool 10.0.0.5:11211 settings # shows settings stats memcached-tool 10.0.0.5:11211 sizes # shows sizes stats memcached-tool 10.0.0.5:11211 dump [limit] # dumps keys and values WARNING! sizes is a development command. As of 1.4 it is still the only command which will lock your memcached instance for some time. If you have many millions of stored items, it can become unresponsive for several minutes. Run this at your own risk. It is roadmapped to either make this feature optional or at least speed it up. " unless $addr && $mode; sub server_connect { my $sock; if ($addr =~ m:/:) { $sock = IO::Socket::UNIX->new( Peer => $addr, ); } else { $addr .= ':11211' unless $addr =~ /:\d+$/; $sock = IO::Socket::INET->new( PeerAddr => $addr, Proto => 'tcp', ); } die "Couldn't connect to $addr\n" unless $sock; return $sock; } my $sock = server_connect(); if ($mode eq 'dump') { print STDERR "Dumping memcache contents"; print STDERR " (limiting to $limit keys)" unless !$limit; print STDERR "\n"; print $sock "lru_crawler metadump all\r\n"; my %keyexp; my $keycount = 0; while (<$sock>) { last if /^END/ or ($limit and $keycount == $limit); # return format looks like this # key=foo exp=2147483647 la=1521046038 cas=717111 fetch=no cls=13 size=1232 if (/^key=(\S+) exp=(-?\d+) .*/) { my ($k, $exp) = ($1, $2); $k =~ s/%(.{2})/chr hex $1/eg; if ($exp == -1) { $keyexp{$k} = 0; } else { $keyexp{$k} = $exp; } } $keycount++; } if ($limit) { # Need to reopen the connection here to stop the metadump in # case the key limit was reached. # # XXX: Once a limit on # of keys returned is introduced in # `lru_crawler metadump`, this should be removed and the proper # parameter passed in the query above. close($sock); $sock = server_connect(); } foreach my $k (keys(%keyexp)) { print $sock "get $k\r\n"; my $response = <$sock>; if ($response =~ /VALUE (\S+) (\d+) (\d+)/) { my $flags = $2; my $len = $3; my $val; read $sock, $val, $len; print "add $k $flags $keyexp{$k} $len\r\n$val\r\n"; # get the END $_ = <$sock>; $_ = <$sock>; } } exit; } if ($mode eq 'stats') { my %items; print $sock "stats\r\n"; while (<$sock>) { last if /^END/; chomp; if (/^STAT\s+(\S*)\s+(.*)/) { $items{$1} = $2; } } printf ("#%-22s %5s %13s\n", $addr, "Field", "Value"); foreach my $name (sort(keys(%items))) { printf ("%29s %14s\n", $name, $items{$name}); } exit; } if ($mode eq 'settings') { my %items; print $sock "stats settings\r\n"; while (<$sock>) { last if /^END/; chomp; if (/^STAT\s+(\S*)\s+(.*)/) { $items{$1} = $2; } } printf ("#%-17s %5s %11s\n", $addr, "Field", "Value"); foreach my $name (sort(keys(%items))) { printf ("%24s %12s\n", $name, $items{$name}); } exit; } if ($mode eq 'sizes') { my %items; print $sock "stats sizes\r\n"; while (<$sock>) { last if /^END/; chomp; if (/^STAT\s+(\S*)\s+(.*)/) { $items{$1} = $2; } } printf ("#%-17s %5s %11s\n", $addr, "Size", "Count"); foreach my $name (sort(keys(%items))) { printf ("%24s %12s\n", $name, $items{$name}); } exit; } # display mode: my %items; # class -> { number, age, chunk_size, chunks_per_page, # total_pages, total_chunks, used_chunks, # free_chunks, free_chunks_end } print $sock "stats items\r\n"; my $max = 0; while (<$sock>) { last if /^END/; if (/^STAT items:(\d+):(\w+) (\d+)/) { $items{$1}{$2} = $3; } } print $sock "stats slabs\r\n"; while (<$sock>) { last if /^END/; if (/^STAT (\d+):(\w+) (\d+)/) { $items{$1}{$2} = $3; $max = $1; } } print " # Item_Size Max_age Pages Count Full? Evicted Evict_Time OOM\n"; foreach my $n (1..$max) { my $it = $items{$n}; next if (0 == $it->{total_pages}); my $size = $it->{chunk_size} < 1024 ? "$it->{chunk_size}B" : sprintf("%.1fK", $it->{chunk_size} / 1024.0); my $full = $it->{used_chunks} == $it->{total_chunks} ? "yes" : " no"; printf("%3d %8s %9ds %7d %7d %7s %8d %8d %4d\n", $n, $size, $it->{age}, $it->{total_pages}, $it->{number}, $full, $it->{evicted}, $it->{evicted_time}, $it->{outofmemory}); } memcached-1.5.22/scripts/damemtop.yaml0000664000175000017500000000044613250303500014606 00000000000000delay: 3 mode: t top_mode: sort_column: "hostname" sort_order: "asc" columns: - hostname - all_version - all_fill_rate - hit_rate - evictions - bytes_written - "2:get_hits" servers: - 127.0.0.1:11211 - 127.0.0.2:11211 memcached-1.5.22/scripts/memcached-init0000775000175000017500000000641313461170555014731 00000000000000#! /bin/bash ### BEGIN INIT INFO # Provides: memcached # Required-Start: $syslog # Required-Stop: $syslog # Should-Start: $local_fs # Should-Stop: $local_fs # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: memcached - Memory caching daemon # Description: memcached - Memory caching daemon ### END INIT INFO # Usage: # cp /etc/memcached.conf /etc/memcached_server1.conf # cp /etc/memcached.conf /etc/memcached_server2.conf # start all instances: # /etc/init.d/memcached start # start one instance: # /etc/init.d/memcached start server1 # stop all instances: # /etc/init.d/memcached stop # stop one instance: # /etc/init.d/memcached stop server1 # There is no "status" command. PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin DAEMON=/usr/bin/memcached DAEMONNAME=memcached DAEMONBOOTSTRAP=/usr/share/memcached/scripts/start-memcached DESC=memcached test -x $DAEMON || exit 0 test -x $DAEMONBOOTSTRAP || exit 0 set -e FILES=(/etc/memcached_*.conf) # check for alternative config schema if [ -r "${FILES[0]}" ]; then CONFIGS=() for FILE in "${FILES[@]}"; do # remove prefix NAME=${FILE#/etc/} # remove suffix NAME=${NAME%.conf} # check optional second param if [ $# -ne 2 ]; then # add to config array CONFIGS+=($NAME) elif [ "memcached_$2" == "$NAME" ]; then # use only one memcached CONFIGS=($NAME) break; fi; done; if [ ${#CONFIGS[@]} == 0 ]; then echo "Config not exist for: $2" >&2 exit 1 fi; else CONFIGS=(memcached) fi; CONFIG_NUM=${#CONFIGS[@]} for ((i=0; i < $CONFIG_NUM; i++)); do NAME=${CONFIGS[${i}]} PIDFILE="/var/run/${NAME}.pid" case "$1" in start) echo -n "Starting $DESC: " start-stop-daemon --start --quiet --exec "$DAEMONBOOTSTRAP" -- /etc/${NAME}.conf $PIDFILE echo "$NAME." ;; stop) echo -n "Stopping $DESC: " start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE --exec $DAEMON echo "$NAME." rm -f $PIDFILE ;; status) [ $# -lt 2 ] && NAME=$DAEMONNAME PIDFILE="/var/run/$NAME.pid" set +e start-stop-daemon --status --pidfile $PIDFILE case $? in 0) echo "$DESC: $NAME (pid $(cat $PIDFILE)) is running" && exit 0;; 1) echo "$DESC: $NAME is not running thro' the pid file exists" && rm -f $PIDFILE && exit 1;; 3) echo "$DESC: $NAME is not running" && exit 3;; 4) echo "$DESC: $NAME status is unclear, sorry" && exit 4;; esac ;; restart|force-reload) # # If the "reload" option is implemented, move the "force-reload" # option to the "reload" entry above. If not, "force-reload" is # just the same as "restart". # echo -n "Restarting $DESC: " start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE rm -f $PIDFILE sleep 1 start-stop-daemon --start --quiet --exec "$DAEMONBOOTSTRAP" -- /etc/${NAME}.conf $PIDFILE echo "$NAME." ;; *) N=/etc/init.d/$NAME # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2 echo "Usage: $N {start|stop|status|restart|force-reload}" >&2 exit 1 ;; esac done; exit 0 memcached-1.5.22/scripts/README.damemtop0000664000175000017500000000117613250303500014602 00000000000000dormando's awesome memcached top A flexible 'top' like utility for viewing memcached clusters. Under development. Latest code is available at: http://github.com/dormando/damemtop See --help for full information. Requires 'AnyEvent', and 'YAML' libraries from CPAN: http://search.cpan.org/ 'AnyEvent' depends on 'common::sense' (also at CPAN). If you have a large cluster and want higher performance, find and install 'EV' from CPAN. AnyEvent will automagically use it and use epoll, kqeueue, etc, for socket handling. Pester me for questions/bugs/ideas. As of writing the util is in early release and missing many future features. memcached-1.5.22/scripts/damemtop0000775000175000017500000003702313250303500013651 00000000000000#!/usr/bin/perl # dormando's awesome memcached top utility! # # Copyright 2009 Dormando (dormando@rydia.net). All rights reserved. # # Use and distribution licensed under the BSD license. See # the COPYING file for full text. use strict; use warnings FATAL => 'all'; use AnyEvent; use AnyEvent::Socket; use AnyEvent::Handle; use Getopt::Long; use YAML qw/Dump Load LoadFile/; use Term::ReadKey qw/ReadMode ReadKey GetTerminalSize/; our $VERSION = '0.1'; my $CLEAR = `clear`; my @TERM_SIZE = (); $|++; my %opts = (); GetOptions(\%opts, 'help|h', 'config=s'); if ($opts{help}) { show_help(); exit; } $SIG{INT} = sub { ReadMode('normal'); print "\n"; exit; }; # TODO: make this load from central location, and merge in homedir changes. # then merge Getopt::Long stuff on top of that # TODO: Set a bunch of defaults and merge in. my $CONF = load_config(); my %CONS = (); my $LAST_RUN = time; # time after the last loop cycle. my $TIME_SINCE_LAST_RUN = time; # time since last loop cycle. my $loop_timer; my $main_cond; my $prev_stats_results; my %display_modes = ( 't' => \&display_top_mode, '?' => \&display_help_mode, 'h' => \&display_help_mode, ); my %column_compute = ( 'hostname' => { stats => [], code => \&compute_hostname}, 'hit_rate' => { stats => ['get_hits', 'get_misses'], code => \&compute_hit_rate }, 'fill_rate' => { stats => ['bytes', 'limit_maxbytes'], code => \&compute_fill_rate }, ); my %column_format = ( 'hit_rate' => \&format_percent, 'fill_rate' => \&format_percent, ); # This can collapse into %column_compute my %column_format_totals = ( 'hit_rate' => 0, 'fill_rate' => 0, ); ReadMode('cbreak'); my $LAST_KEY = ''; my $read_keys = AnyEvent->io ( fh => \*STDIN, poll => 'r', cb => sub { $LAST_KEY = ReadKey(-1); # If there is a running timer, cancel it. # Don't want to interrupt a main loop run. # fire_main_loop()'s iteration will pick up the keypress. if ($loop_timer) { $loop_timer = undef; $main_cond->send; } } ); # start main loop fire_main_loop(); ### AnyEvent related code. sub fire_main_loop { for (;;) { $loop_timer = undef; $main_cond = AnyEvent->condvar; my $time_taken = main_loop(); my $delay = $CONF->{delay} - $time_taken; $delay = 0 if $delay < 0; $loop_timer = AnyEvent->timer( after => $delay, cb => $main_cond, ); $main_cond->recv; } } sub main_loop { my $start = AnyEvent->now; # use ->time to find the end. maintain_connections(); my $cv = AnyEvent->condvar; # FIXME: Need to dump early if there're no connected conns # FIXME: Make this only fetch stats from cons we care to visualize? # maybe keep everything anyway to maintain averages? my %stats_results = (); while (my ($hostname, $con) = each %CONS) { $cv->begin; call_stats($con, ['', 'items', 'slabs'], sub { $stats_results{$hostname} = shift; $cv->end; }); } $cv->recv; # Short circuit since we don't have anything to compare to. unless ($prev_stats_results) { $prev_stats_results = \%stats_results; return $CONF->{delay}; } # Semi-exact global time diff for stats that want to average # themselves per-second. my $this_run = AnyEvent->time; $TIME_SINCE_LAST_RUN = $this_run - $LAST_RUN; $LAST_RUN = $this_run; # Done all our fetches. Drive the display. display_run($prev_stats_results, \%stats_results); $prev_stats_results = \%stats_results; my $end = AnyEvent->time; my $diff = $LAST_RUN - $start; print "loop took: $diff"; return $diff; } sub maintain_connections { my $cv = AnyEvent->condvar; $cv->begin (sub { shift->send }); for my $host (@{$CONF->{servers}}) { next if $CONS{$host}; $cv->begin; $CONS{$host} = connect_memcached($host, sub { if ($_[0] eq 'err') { print "Failed connecting to $host: ", $_[1], "\n"; delete $CONS{$host}; } $cv->end; }); } $cv->end; $cv->recv; } sub connect_memcached { my ($fullhost, $cb) = @_; my ($host, $port) = split /:/, $fullhost; my $con; $con = AnyEvent::Handle->new ( connect => [$host => $port], on_connect => sub { $cb->('con'); }, on_connect_error => sub { $cb->('err', $!); $con->destroy; }, on_eof => sub { $cb->('err', $!); $con->destroy; }, ); return $con; } # Function's getting a little weird since I started optimizing it. # As of my first set of production tests, this routine is where we spend # almost all of our processing time. sub call_stats { my ($con, $cmds, $cb) = @_; my $stats = {}; my $num_types = @$cmds; my $reader; $reader = sub { my ($con, $results) = @_; { my %temp = (); for my $line (split(/\n/, $results)) { my ($k, $v) = (split(/\s+/, $line))[1,2]; $temp{$k} = $v; } $stats->{$cmds->[0]} = \%temp; } shift @$cmds; unless (@$cmds) { # Out of commands to process, return goodies. $cb->($stats); return; } }; for my $cmd (@$cmds) { $con->push_write('stats ' . $cmd . "\n"); $stats->{$cmd} = {}; $con->push_read(line => "END\r\n", $reader); } } ### Compute routines sub compute_hostname { return $_[0]; } sub compute_hit_rate { my $s = $_[1]; my $total = $s->{get_hits} + $s->{get_misses}; return 'NA' unless $total; return $s->{get_hits} / $total; } sub compute_fill_rate { my $s = $_[1]; return $s->{bytes} / $s->{limit_maxbytes}; } sub format_column { my ($col, $val) = @_; my $res; $col =~ s/^all_//; if ($column_format{$col}) { if (ref($column_format{$col}) eq 'CODE') { return $column_format{$col}->($val); } else { return $val .= $column_format{$col}; } } else { return format_commas($val); } } sub column_can_total { my $col = shift; $col =~ s/^all_//; return 1 unless exists $column_format_totals{$col}; return $column_format_totals{$col}; } ### Display routines # If there isn't a specific column type computer, see if we just want to # look at the specific stat and return it. # If column is a generic type and of 'all_cmd_get' format, return the more # complete stat instead of the diffed stat. sub compute_column { my ($col, $host, $prev_stats, $curr_stats) = @_; my $diff_stats = 1; $diff_stats = 0 if ($col =~ s/^all_//); # Really should decide on whether or not to flatten the hash :/ my $find_stat = sub { for my $type (keys %{$_[0]}) { return $_[0]->{$type}->{$_[1]} if exists $_[0]->{$type}->{$_[1]}; } }; my $diff_stat = sub { my $stat = shift; return 'NA' unless defined $find_stat->($curr_stats, $stat); if ($diff_stats) { my $diff = eval { return ($find_stat->($curr_stats, $stat) - $find_stat->($prev_stats, $stat)) / $TIME_SINCE_LAST_RUN; }; return 'NA' if ($@); return $diff; } else { return $find_stat->($curr_stats, $stat); } }; if (my $comp = $column_compute{$col}) { my %s = (); for my $stat (@{$comp->{stats}}) { $s{$stat} = $diff_stat->($stat); } return $comp->{code}->($host, \%s); } else { return $diff_stat->($col); } return 'NA'; } # We have a bunch of stats from a bunch of connections. # At this point we run a particular display mode, capture the lines, then # truncate and display them. sub display_run { my $prev_stats = shift; my $curr_stats = shift; @TERM_SIZE = GetTerminalSize; die "cannot detect terminal size" unless $TERM_SIZE[0] && $TERM_SIZE[1]; if ($LAST_KEY eq 'q') { print "\n"; ReadMode('normal'); exit; } if ($LAST_KEY ne $CONF->{mode} && exists $display_modes{$LAST_KEY}) { $CONF->{prev_mode} = $CONF->{mode}; $CONF->{mode} = $LAST_KEY; } elsif ($CONF->{mode} eq 'h' || $CONF->{mode} eq '?') { # Bust out of help mode on any key. $CONF->{mode} = $CONF->{prev_mode}; } my $lines = $display_modes{$CONF->{mode}}->($prev_stats, $curr_stats); display_lines($lines) if $lines; } # Default "top" mode. # create a set of computed columns as requested by the config. # this has gotten a little out of hand... needs more cleanup/abstraction. sub display_top_mode { my $prev_stats = shift; my $curr_stats = shift; my @columns = @{$CONF->{top_mode}->{columns}}; my @rows = (); my @tot_row = (); # Round one. for my $host (sort keys %{$curr_stats}) { my @row = (); for my $colnum (0 .. @columns-1) { my $col = $columns[$colnum]; my $res = compute_column($col, $host, $prev_stats->{$host}, $curr_stats->{$host}); $tot_row[$colnum] += $res if is_numeric($res); push @row, $res; } push(@rows, \@row); } # Sort rows by sort column (ascending or descending) if (my $sort = $CONF->{top_mode}->{sort_column}) { my $order = $CONF->{top_mode}->{sort_order} || 'asc'; my $colnum = 0; for (0 .. @columns-1) { $colnum = $_ if $columns[$_] eq $sort; } my @newrows; if ($order eq 'asc') { if (is_numeric($rows[0]->[$colnum])) { @newrows = sort { $a->[$colnum] <=> $b->[$colnum] } @rows; } else { @newrows = sort { $a->[$colnum] cmp $b->[$colnum] } @rows; } } else { if (is_numeric($rows[0]->[$colnum])) { @newrows = sort { $b->[$colnum] <=> $a->[$colnum] } @rows; } else { @newrows = sort { $b->[$colnum] cmp $a->[$colnum] } @rows; } } @rows = @newrows; } # Format each column after the sort... { my @newrows = (); for my $row (@rows) { my @newrow = (); for my $colnum (0 .. @columns-1) { push @newrow, is_numeric($row->[$colnum]) ? format_column($columns[$colnum], $row->[$colnum]) : $row->[$colnum]; } push @newrows, \@newrow; } @rows = @newrows; } # Create average and total rows. my @avg_row = (); for my $col (0 .. @columns-1) { if (is_numeric($tot_row[$col])) { my $countable_rows = 0; for my $row (@rows) { next unless $row->[$col]; $countable_rows++ unless $row->[$col] eq 'NA'; } $countable_rows = 1 unless $countable_rows; push @avg_row, format_column($columns[$col], sprintf('%.2f', $tot_row[$col] / $countable_rows)); } else { push @avg_row, 'NA'; } $tot_row[$col] = 'NA' unless defined $tot_row[$col]; $tot_row[$col] = 'NA' unless (column_can_total($columns[$col])); $tot_row[$col] = format_column($columns[$col], $tot_row[$col]) unless $tot_row[$col] eq 'NA'; } unshift @rows, \@avg_row; unshift @rows, ['AVERAGE:']; unshift @rows, \@tot_row; unshift @rows, ['TOTAL:']; # Round two. Pass @rows into a function which returns an array with the # desired format spacing for each column. unshift @rows, \@columns; my $spacing = find_optimal_spacing(\@rows); my @display_lines = (); for my $row (@rows) { my $line = ''; for my $col (0 .. @$row-1) { my $space = $spacing->[$col]; $line .= sprintf("%-${space}s ", $row->[$col]); } push @display_lines, $line; } return \@display_lines; } sub display_help_mode { my $help = <<"ENDHELP"; dormando's awesome memcached top utility version v$VERSION This early version requires you to edit the ~/.damemtop/damemtop.yaml (or /etc/damemtop.yaml) file in order to change options. See --help for more info. Hit any key to exit help. ENDHELP my @lines = split /\n/, $help; display_lines(\@lines); $LAST_KEY = ReadKey(0); return; } # Takes a set of lines, clears screen, dumps header, trims lines, etc # MAYBE: mode to wrap lines instead of trim them? sub display_lines { my $lines = shift; my $width = $TERM_SIZE[0]; my $height_remain = $TERM_SIZE[1]; unshift @$lines, display_header($width); clear_screen() unless $CONF->{no_clear}; while (--$height_remain && @$lines) { # truncate too long lines. my $line = shift @$lines; $line = substr $line, 0, $width-1; print $line, "\n"; } } sub display_header { my $topbar = 'damemtop: ' . scalar localtime; if ($CONF->{mode} eq 't' && $CONF->{top_mode}->{sort_column}) { $topbar .= ' [sort: ' . $CONF->{top_mode}->{sort_column} . ']'; } $topbar .= ' [delay: ' . $CONF->{delay} . 's]'; return $topbar; } ### Utilities # find the optimal format spacing for each column, which is: # longest length of item in col + 2 (whitespace). sub find_optimal_spacing { my $rows = shift; my @maxes = (); my $num_cols = @{$rows->[0]}; for my $row (@$rows) { for my $col (0 .. $num_cols-1) { $maxes[$col] = 0 unless $maxes[$col]; next unless $row->[$col]; $maxes[$col] = length($row->[$col]) if length($row->[$col]) > $maxes[$col]; } } for my $col (0 .. $num_cols) { $maxes[$col] += 1; } return \@maxes; } # doesn't try too hard to identify numbers... sub is_numeric { return 0 unless $_[0]; return 1 if $_[0] =~ m/^\d+(\.\d*)?(\w+)?$/; return 0; } sub format_percent { return sprintf("%.2f%%", $_[0] * 100); } sub format_commas { my $num = shift; $num = int($num); $num =~ s/(^[-+]?\d+?(?=(?>(?:\d{3})+)(?!\d))|\G\d{3}(?=\d))/$1,/g; return $num; } # Can tick counters/etc here as well. sub clear_screen { print $CLEAR; } # tries minimally to find a localized config file. # TODO: Handle the YAML error and make it prettier. sub load_config { my $config = $opts{config} if $opts{config}; my $homedir = "$ENV{HOME}/.damemtop/damemtop.yaml"; if (-e $homedir) { $config = $homedir; } else { $config = '/etc/damemtop.yaml'; } return LoadFile($config); } sub show_help { print <<"ENDHELP"; dormando's awesome memcached top utility version v$VERSION This program is copyright (c) 2009 Dormando. Use and distribution licensed under the BSD license. See the COPYING file for full text. contact: dormando\@rydia.net or memcached\@googlegroups.com. This early version requires you to edit the ~/.damemtop/damemtop.yaml (or /etc/damemtop.yaml) file in order to change options. You may display any column that is in the output of 'stats', 'stats items', or 'stats slabs' from memcached's ASCII protocol. Start a column with 'all_' (ie; 'all_get_hits') to display the current stat, otherwise the stat is displayed as an average per second. Specify a "sort_column" under "top_mode" to sort the output by any column. Some special "computed" columns exist: hit_rate (get/miss hit ratio) fill_rate (% bytes used out of the maximum memory limit) ENDHELP exit; } memcached-1.5.22/scripts/memcached.service0000664000175000017500000000651413461170555015426 00000000000000# It's not recommended to modify this file in-place, because it will be # overwritten during upgrades. If you want to customize, the best # way is to use the "systemctl edit" command to create an override unit. # # For example, to pass additional options, create an override unit # (as is done by systemctl edit) and enter the following: # # [Service] # Environment=OPTIONS="-l 127.0.0.1,::1" [Unit] Description=memcached daemon After=network.target [Service] EnvironmentFile=/etc/sysconfig/memcached ExecStart=/usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS # Set up a new file system namespace and mounts private /tmp and /var/tmp # directories so this service cannot access the global directories and # other processes cannot access this service's directories. PrivateTmp=true # Mounts the /usr, /boot, and /etc directories read-only for processes # invoked by this unit. ProtectSystem=full # Ensures that the service process and all its children can never gain new # privileges NoNewPrivileges=true # Sets up a new /dev namespace for the executed processes and only adds API # pseudo devices such as /dev/null, /dev/zero or /dev/random (as well as # the pseudo TTY subsystem) to it, but no physical devices such as /dev/sda. PrivateDevices=true # Required for dropping privileges and running as a different user CapabilityBoundingSet=CAP_SETGID CAP_SETUID CAP_SYS_RESOURCE # Restricts the set of socket address families accessible to the processes # of this unit. Protects against vulnerabilities such as CVE-2016-8655 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX # Some security features are not in the older versions of systemd used by # e.g. RHEL7/CentOS 7. The below settings are automatically edited at package # build time to uncomment them if the target platform supports them. # Attempts to create memory mappings that are writable and executable at # the same time, or to change existing memory mappings to become executable # are prohibited. ##safer##MemoryDenyWriteExecute=true # Explicit module loading will be denied. This allows to turn off module # load and unload operations on modular kernels. It is recommended to turn # this on for most services that do not need special file systems or extra # kernel modules to work. ##safer##ProtectKernelModules=true # Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, # /proc/latency_stats, /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq # will be made read-only to all processes of the unit. Usually, tunable # kernel variables should only be written at boot-time, with the sysctl.d(5) # mechanism. Almost no services need to write to these at runtime; it is hence # recommended to turn this on for most services. ##safer##ProtectKernelTunables=true # The Linux Control Groups (cgroups(7)) hierarchies accessible through # /sys/fs/cgroup will be made read-only to all processes of the unit. # Except for container managers no services should require write access # to the control groups hierarchies; it is hence recommended to turn this # on for most services ##safer##ProtectControlGroups=true # Any attempts to enable realtime scheduling in a process of the unit are # refused. ##safer##RestrictRealtime=true # Takes away the ability to create or manage any kind of namespace ##safer##RestrictNamespaces=true [Install] WantedBy=multi-user.target memcached-1.5.22/scripts/memcached@.service0000664000175000017500000000713313461170555015524 00000000000000# It's not recommended to modify this file in-place, because it will be # overwritten during upgrades. If you want to customize, the best # way is to use the "systemctl edit" command to create an override unit. # # For example, to pass additional options, create an override unit # (as is done by systemctl edit) and enter the following: # # [Service] # Environment=OPTIONS="-l 127.0.0.1,::1" # # To use the "instanced" version of this, just start 'memcached@11211' or # whatever port you'd like. If /etc/sysconfig/memcached. exists, it # will be read first, so you can set different parameters for a given # instance. [Unit] Description=memcached daemon After=network.target [Service] EnvironmentFile=/etc/sysconfig/memcached EnvironmentFile=-/etc/sysconfig/memcached.%i ExecStart=/usr/bin/memcached -p %i -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS # Set up a new file system namespace and mounts private /tmp and /var/tmp # directories so this service cannot access the global directories and # other processes cannot access this service's directories. PrivateTmp=true # Mounts the /usr, /boot, and /etc directories read-only for processes # invoked by this unit. ProtectSystem=full # Ensures that the service process and all its children can never gain new # privileges NoNewPrivileges=true # Sets up a new /dev namespace for the executed processes and only adds API # pseudo devices such as /dev/null, /dev/zero or /dev/random (as well as # the pseudo TTY subsystem) to it, but no physical devices such as /dev/sda. PrivateDevices=true # Required for dropping privileges and running as a different user CapabilityBoundingSet=CAP_SETGID CAP_SETUID CAP_SYS_RESOURCE # Restricts the set of socket address families accessible to the processes # of this unit. Protects against vulnerabilities such as CVE-2016-8655 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX # Some security features are not in the older versions of systemd used by # e.g. RHEL7/CentOS 7. The below settings are automatically edited at package # build time to uncomment them if the target platform supports them. # Attempts to create memory mappings that are writable and executable at # the same time, or to change existing memory mappings to become executable # are prohibited. ##safer##MemoryDenyWriteExecute=true # Explicit module loading will be denied. This allows to turn off module # load and unload operations on modular kernels. It is recommended to turn # this on for most services that do not need special file systems or extra # kernel modules to work. ##safer##ProtectKernelModules=true # Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, # /proc/latency_stats, /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq # will be made read-only to all processes of the unit. Usually, tunable # kernel variables should only be written at boot-time, with the sysctl.d(5) # mechanism. Almost no services need to write to these at runtime; it is hence # recommended to turn this on for most services. ##safer##ProtectKernelTunables=true # The Linux Control Groups (cgroups(7)) hierarchies accessible through # /sys/fs/cgroup will be made read-only to all processes of the unit. # Except for container managers no services should require write access # to the control groups hierarchies; it is hence recommended to turn this # on for most services ##safer##ProtectControlGroups=true # Any attempts to enable realtime scheduling in a process of the unit are # refused. ##safer##RestrictRealtime=true # Takes away the ability to create or manage any kind of namespace ##safer##RestrictNamespaces=true [Install] WantedBy=multi-user.target memcached-1.5.22/scripts/memcached-automove0000775000175000017500000001771513461170555015634 00000000000000#!/usr/bin/python3 # Copyright 2017 Facebook. # Licensed under the same terms as memcached itself. import argparse import socket import sys import re import traceback from time import sleep parser = argparse.ArgumentParser(description="daemon for rebalancing slabs") parser.add_argument("--host", help="host to connect to", default="localhost:11211", metavar="HOST:PORT") parser.add_argument("-s", "--sleep", help="seconds between runs", type=int, default="1") parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("-a", "--automove", action="store_true", default=False, help="enable automatic page rebalancing") parser.add_argument("-w", "--window", type=int, default="30", help="rolling window size for decision history") parser.add_argument("-r", "--ratio", type=float, default=0.8, help="ratio limiting distance between low/high class ages") # TODO: # - age adjuster function # - by ratio of get_hits # - by ratio of chunk size args = parser.parse_args() host, port = args.host.split(':') def window_check(history, sid, key): total = 0 for window in history['w']: s = window.get(sid) if s and s.get(key): total += s.get(key) return total def determine_move(history, diffs, totals): """ Figure out of a page move is in order. - if > 2.5 pages of free space without free chunks reducing for N trials, and no evictions for N trials, free to global. - use ratio of how far apart age can be between slab classes - TODO: if get_hits exists, allowable distance in age from the *youngest* slab class is based on the percentage of get_hits the class gets, against the factored max distance, ie: 1% max delta. youngest is 900, oldest allowable is 900+90 if class has 30% of get_hits, it can be 930 - youngest evicting slab class gets page moved to it, if outside ratio max - use age as average over window. smooths over items falling out of WARM. also gives a little weight: if still evicting, a few more pages than necessary may be moved, pulling the classes closer together. Hopefully avoiding ping-ponging. """ # rotate windows history['w'].append({}) if (len(history['w']) > args.window): history['w'].pop(0) w = history['w'][-1] oldest = (-1, 0) youngest = (-1, sys.maxsize) decision = (-1, -1) for sid, slab in diffs.items(): w[sid] = {} if 'evicted_d' not in slab or 'total_pages_d' not in slab: continue # mark this window as dirty if total pages increases or evictions # happened if slab['total_pages_d'] > 0: w[sid]['dirty'] = 1 if slab['evicted_d'] > 0: w[sid]['dirty'] = 1 w[sid]['ev'] = 1 w[sid]['age'] = slab['age'] age = window_check(history, sid, 'age') / len(history['w']) # if > 2.5 pages free, and not dirty, reassign to global page pool and # break. if slab['free_chunks'] > slab['chunks_per_page'] * 2.5: if window_check(history, sid, 'dirty') == 0: decision = (sid, 0) break # are we the oldest slab class? (and a valid target) if age > oldest[1] and slab['total_pages'] > 2: oldest = (sid, age) # are we the youngest evicting slab class? ev_total = window_check(history, sid, 'ev') window_min = args.window / 2 if age < youngest[1] and ev_total > window_min: youngest = (sid, age) #if args.verbose: # print("window: {} range: {}".format(ev_total, window_min)) # is the youngest slab class too young? if youngest[0] != -1 and oldest[0] != -1: if args.verbose: print("old: [class: {}] [age: {:.2f}]\nyoung: [class: {}] [age: {:.2f}]".format( int(oldest[0]), oldest[1], int(youngest[0]), youngest[1])) if youngest[1] < oldest[1] * args.ratio and w[youngest[0]].get('ev'): decision = (oldest[0], youngest[0]) if (len(history['w']) >= args.window): return decision return (-1, -1) def run_move(s, decision): s.write("slabs reassign " + str(decision[0]) + " " + str(decision[1]) + "\r\n") line = s.readline().rstrip() if args.verbose: print("move result:", line) def diff_stats(before, after): """ fills out "diffs" as deltas between before/after, and "totals" as the sum of all slab classes. "_d" postfix to keys means the delta between before/after. non-postfix keys are total as of 'after's reading. """ diffs = {} totals = {} for slabid in after.keys(): sb = before.get(slabid) sa = after.get(slabid) if not (sb and sa): continue slab = sa.copy() for k in sa.keys(): if k not in sb: continue if k not in totals: totals[k] = 0 totals[k + '_d'] = 0 if k + '_d' not in slab: slab[k + '_d'] = 0 if re.search(r"^\d+$", sa[k]): totals[k] += int(sa[k]) slab[k] = int(sa[k]) slab[k + '_d'] = int(sa[k]) - int(sb[k]) totals[k + '_d'] += int(sa[k]) - int(sb[k]) slab['slab'] = slabid diffs[slabid] = slab return (diffs, totals) def read_stats(s): slabs = {} for statcmd in ['items', 'slabs']: #print("stat cmd: " + statcmd) # FIXME: Formatting s.write("stats " + statcmd + "\r\n") while True: line = s.readline().rstrip() if line.startswith("END"): break m = re.match(r"^STAT (?:items:)?(\d+):(\S+) (\S+)", line) if m: (slab, var, val) = m.groups() if slab not in slabs: slabs[slab] = {} slabs[slab][var] = val #print("line: " + line) return slabs def pct(num, divisor): if not divisor: return 0 return (num / divisor) def show_detail(diffs, totals): """ just a pretty printer for some extra data """ print("\n {:2s}: {:8s} (pct ) {:10s} (pct ) {:6s} (pct) {:6s}".format('sb', 'evicted', 'items', 'pages', 'age')) for sid, slab in diffs.items(): if 'evicted_d' not in slab: continue print(" {:2d}: {:8d} ({:.2f}%) {:10d} ({:.4f}%) {:6d} ({:.2f}%) {:6d}".format( int(sid), slab['evicted_d'], pct(slab['evicted_d'], totals['evicted_d']), slab['number'], pct(slab['number'], totals['number']), slab['total_pages'], pct(slab['total_pages'], totals['total_pages']), slab['age'])) stats_pre = {} history = { 'w': [{}] } while True: try: with socket.create_connection((host, port), 5) as c: s = c.makefile(mode="rw", buffering=1) s.write("slabs automove 0\r\n") print(s.readline().rstrip()) while True: stats_post = read_stats(s) (diffs, totals) = diff_stats(stats_pre, stats_post) if args.verbose: show_detail(diffs, totals) decision = determine_move(history, diffs, totals) if decision[0] != -1 and decision[1] != -1: print("moving page from, to:", decision) if args.automove: run_move(s, decision) # Minimize sleeping if we just moved a page to global pool. # Improves responsiveness during flushes/quick changes. if decision[1] == 0: sleep(0.05) else: sleep(args.sleep) stats_pre = stats_post except: err = sys.exc_info() print("disconnected:", err[0], err[1]) traceback.print_exc() stats_pre = {} history = { 'w': [{}] } sleep(args.sleep) memcached-1.5.22/scripts/memcached.upstart0000664000175000017500000000107313461170555015463 00000000000000description "memcached - high performance memory caching daemon" author "Cameron Norman " start on filesystem stop on runlevel [016] pre-start script ret=0 test -x /usr/bin/memcached || { stop; exit 0; } # Main start memcached-server || ret=$? # Instances for i in /etc/memcached_*.conf; do i=${i#/etc/memcached_} i=${i%.conf} test "$i" = "*" && continue start memcached-server SERVER=$i || ret=$? done exit $ret end script post-stop exec initctl emit stop-memcached-servers memcached-1.5.22/scripts/memcached.sysv0000775000175000017500000000260613461170555014773 00000000000000#! /bin/sh # # chkconfig: - 55 45 # description: The memcached daemon is a network memory cache service. # processname: memcached # config: /etc/sysconfig/memcached # Source function library. . /etc/rc.d/init.d/functions PORT=11211 USER=nobody MAXCONN=1024 CACHESIZE=64 OPTIONS="" if [ -f /etc/sysconfig/memcached ];then . /etc/sysconfig/memcached fi # Check that networking is up. if [ "$NETWORKING" = "no" ] then exit 0 fi RETVAL=0 prog="memcached" start () { echo -n $"Starting $prog: " # insure that /var/run/memcached has proper permissions chown $USER /var/run/memcached daemon memcached -d -p $PORT -u $USER -m $CACHESIZE -c $MAXCONN -P /var/run/memcached/memcached.pid $OPTIONS RETVAL=$? echo [ $RETVAL -eq 0 ] && touch /var/lock/subsys/memcached } stop () { echo -n $"Stopping $prog: " killproc memcached RETVAL=$? echo if [ $RETVAL -eq 0 ] ; then rm -f /var/lock/subsys/memcached rm -f /var/run/memcached/memcached.pid fi } restart () { stop start } # See how we were called. case "$1" in start) start ;; stop) stop ;; status) status memcached ;; restart|reload) restart ;; condrestart) [ -f /var/lock/subsys/memcached ] && restart || : ;; *) echo $"Usage: $0 {start|stop|status|restart|reload|condrestart}" exit 1 esac exit $? memcached-1.5.22/scripts/memcached-tool.10000664000175000017500000000354613575022500015073 00000000000000.TH MEMCACHED-TOOL 1 "Jul 2, 2013" .SH NAME memcached-tool \- stats and management tool for memcached .SH SYNOPSIS .B memcached-tool .RI < host [: port "] | " /path/to/socket "> [" mode ] .SH DESCRIPTION .B memcached-tool is a Perl script used to print statistics from a running memcached instance. The first parameter specifies the address of the daemon either by a hostname, optionally followed by the port number (the default is 11211), or a path to UNIX domain socket. The second parameter specifies the mode in which the tool should run. .SH MODES .TP .B display Print slab class statistics. This is the default mode if no mode is specified. The printed columns are: .RS .TP .B # Number of the slab class. .TP .B Item_Size The amount of space each chunk uses. One item uses one chunk of the appropriate size. .TP .B Max_age Age of the oldest item in the LRU. .TP .B Pages Total number of pages allocated to the slab class. .TP .B Count Number of items presently stored in this class. Expired items are not automatically excluded. .TP .B Full? Yes if there are no free chunks at the end of the last allocated page. .TP .B Evicted Number of times an item had to be evicted from the LRU before it expired. .TP .B Evict_Time Seconds since the last access for the most recent item evicted from this class. .TP .B OOM Number of times the underlying slab class was unable to store a new item. .RE .TP .B stats Print general-purpose statistics of the daemon. Each line contains the name of the statistic and its value. .TP .B dump [limit] Make a partial dump of the cache written in the add statements of the memcached protocol. If .B limit is given and is a strictly positive integer, then the dump is limited to that number of items. .SH SEE ALSO .BR memcached (1), .br .B https://www.memcached.org .SH AUTHOR The memcached-tool script was written by Brad Fitzpatrick .B memcached-1.5.22/scripts/memcached-server.upstart0000664000175000017500000000131413461170555016765 00000000000000description "memcached-server - high-performance memory caching daemon (instance)" author "Cameron Norman " stop on stop-memcached-servers or runlevel [016] expect daemon respawn # Instance w/ default value so main server starts without SERVER param env SERVER="" instance $SERVER usage "SERVER - instance of memcached with corresponding /etc/memcached_$SERVER config" script if test "x$SERVER" = "x"; then name="memcached" else name="memcached_$SERVER" fi test -f /etc/${name}.conf || { echo "No config found, not starting."; exit 1 } exec /usr/share/memcached/scripts/start-memcached /etc/${name}.conf /var/run/${name}.pid end script memcached-1.5.22/scripts/memcached.sysconfig0000664000175000017500000000047113461170555015766 00000000000000# These defaults will be used by every memcached instance, unless overridden # by values in /etc/sysconfig/memcached. USER="nobody" MAXCONN="1024" CACHESIZE="64" OPTIONS="" # The PORT variable will only be used by memcached.service, not by # memcached@xxxxx services, which will use the xxxxx PORT="11211" memcached-1.5.22/bipbuffer.c0000664000175000017500000000726413461170555012562 00000000000000/** * Copyright (c) 2011, Willem-Hendrik Thiart * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE.bipbuffer file. * * @file * @author Willem Thiart himself@willemthiart.com */ #include "stdio.h" #include /* for memcpy */ #include #include "bipbuffer.h" static size_t bipbuf_sizeof(const unsigned int size) { return sizeof(bipbuf_t) + size; } int bipbuf_unused(const bipbuf_t* me) { if (1 == me->b_inuse) /* distance between region B and region A */ return me->a_start - me->b_end; else return me->size - me->a_end; } int bipbuf_size(const bipbuf_t* me) { return me->size; } int bipbuf_used(const bipbuf_t* me) { return (me->a_end - me->a_start) + me->b_end; } void bipbuf_init(bipbuf_t* me, const unsigned int size) { me->a_start = me->a_end = me->b_end = 0; me->size = size; me->b_inuse = 0; } bipbuf_t *bipbuf_new(const unsigned int size) { bipbuf_t *me = malloc(bipbuf_sizeof(size)); if (!me) return NULL; bipbuf_init(me, size); return me; } void bipbuf_free(bipbuf_t* me) { free(me); } int bipbuf_is_empty(const bipbuf_t* me) { return me->a_start == me->a_end; } /* find out if we should turn on region B * ie. is the distance from A to buffer's end less than B to A? */ static void __check_for_switch_to_b(bipbuf_t* me) { if (me->size - me->a_end < me->a_start - me->b_end) me->b_inuse = 1; } /* TODO: DOCUMENT THESE TWO FUNCTIONS */ unsigned char *bipbuf_request(bipbuf_t* me, const int size) { if (bipbuf_unused(me) < size) return 0; if (1 == me->b_inuse) { return (unsigned char *)me->data + me->b_end; } else { return (unsigned char *)me->data + me->a_end; } } int bipbuf_push(bipbuf_t* me, const int size) { if (bipbuf_unused(me) < size) return 0; if (1 == me->b_inuse) { me->b_end += size; } else { me->a_end += size; } __check_for_switch_to_b(me); return size; } int bipbuf_offer(bipbuf_t* me, const unsigned char *data, const int size) { /* not enough space */ if (bipbuf_unused(me) < size) return 0; if (1 == me->b_inuse) { memcpy(me->data + me->b_end, data, size); me->b_end += size; } else { memcpy(me->data + me->a_end, data, size); me->a_end += size; } __check_for_switch_to_b(me); return size; } unsigned char *bipbuf_peek(const bipbuf_t* me, const unsigned int size) { /* make sure we can actually peek at this data */ if (me->size < me->a_start + size) return NULL; if (bipbuf_is_empty(me)) return NULL; return (unsigned char *)me->data + me->a_start; } unsigned char *bipbuf_peek_all(const bipbuf_t* me, unsigned int *size) { if (bipbuf_is_empty(me)) return NULL; *size = me->a_end - me->a_start; return (unsigned char*)me->data + me->a_start; } unsigned char *bipbuf_poll(bipbuf_t* me, const unsigned int size) { if (bipbuf_is_empty(me)) return NULL; /* make sure we can actually poll this data */ if (me->size < me->a_start + size) return NULL; void *end = me->data + me->a_start; me->a_start += size; /* we seem to be empty.. */ if (me->a_start == me->a_end) { /* replace a with region b */ if (1 == me->b_inuse) { me->a_start = 0; me->a_end = me->b_end; me->b_end = me->b_inuse = 0; } else /* safely move cursor back to the start because we are empty */ me->a_start = me->a_end = 0; } __check_for_switch_to_b(me); return end; } memcached-1.5.22/t/0000755000175000017500000000000013615376577010777 500000000000000memcached-1.5.22/t/ascii-auth.t0000664000175000017500000000250113575022500013106 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 9; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; # FIXME: Some tests are forcing UDP to be enabled via MemcachedTest.pm - need # to audit and fix. my $server = new_memcached("-Y $Bin/authfile -U 0"); my $sock = $server->sock; # Test unauthenticated modes print $sock "set foo 0 0 2\r\nhi\r\n"; like(scalar <$sock>, qr/CLIENT_ERROR/, "failed to do a write"); print $sock "get foo\r\n"; like(scalar <$sock>, qr/CLIENT_ERROR/, "failed to do a read"); # Fail to authenticate. print $sock "set foo 0 0 7\r\nfoo bab\r\n"; like(scalar <$sock>, qr/CLIENT_ERROR/, "failed to authenticate"); # Try for real. print $sock "set foo 0 0 7\r\nfoo bar\r\n"; like(scalar <$sock>, qr/STORED/, "authenticated?"); print $sock "set toast 0 0 2\r\nhi\r\n"; like(scalar <$sock>, qr/STORED/, "stored an item that didn't look like user/pass"); mem_get_is($sock, "toast", "hi"); # Create a second socket, try to authenticate against the second token. my $sock2 = $server->new_sock; print $sock2 "set foo 0 0 10\r\nbaaaz quux\r\n"; like(scalar <$sock2>, qr/STORED/, "authenticated a second socket?"); print $sock2 "set toast2 0 0 2\r\nho\r\n"; like(scalar <$sock2>, qr/STORED/, "stored an item that didn't look like user/pass"); mem_get_is($sock2, "toast2", "ho"); # TODO: tests for reloads. memcached-1.5.22/t/idle-timeout.t0000664000175000017500000000222113575022500013457 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More tests => 11; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; # start up a server with 10 maximum connections my $server = new_memcached("-o idle_timeout=3 -l 127.0.0.1"); my $sock = $server->sock; # Make sure we can talk to start with my $stats = mem_stats($sock); is($stats->{idle_kicks}, "0", "check stats initial"); isnt($sock->connected(), undef, "check connected"); # Make sure we don't timeout when active for (my $i = 0; $i < 6; $i++) { $stats = mem_stats($sock); isnt($stats->{version}, undef, "check active $i"); } $stats = mem_stats($sock); is($stats->{idle_kicks}, "0", "check stats 2"); # Make sure we do timeout when not sleep(5); mem_stats($sock); # Network activity, so socket code will see dead socket sleep(1); # we run SSL tests over TCP; hence IO::Socket::SSL returns # '' upon disconnecting with the server. if (enabled_tls_testing()) { is($sock->connected(),'', "check disconnected"); } else { is($sock->connected(),undef, "check disconnected"); } $sock = $server->sock; $stats = mem_stats($sock); isnt($stats->{idle_kicks}, 0, "check stats timeout"); memcached-1.5.22/t/slabs-reassign2.t0000664000175000017500000000747413461170555014104 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More tests => 12; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use Data::Dumper qw/Dumper/; my $server = new_memcached('-m 60 -o slab_reassign,slab_automove,lru_crawler,lru_maintainer,slab_automove_window=3'); my $sock = $server->sock; my $value = "B"x11000; my $keycount = 5000; my $res; for (1 .. $keycount) { print $sock "set nfoo$_ 0 0 11000 noreply\r\n$value\r\n"; } my $todelete = 0; { my $stats = mem_stats($sock); cmp_ok($stats->{curr_items}, '>', 4000, "stored at least 4000 11k items"); $todelete = $stats->{curr_items}; # for ('evictions', 'reclaimed', 'curr_items', 'cmd_set', 'bytes') { # print STDERR "$_: ", $stats->{$_}, "\n"; # } } # Make room in old class so rescues can happen when we switch slab classes. for (1 .. $todelete) { next unless $_ % 2 == 0; print $sock "delete nfoo$_ noreply\r\n"; } { my $tries; for ($tries = 20; $tries > 0; $tries--) { sleep 1; my $stats = mem_stats($sock); if ($stats->{slab_global_page_pool} > 24) { last; } } cmp_ok($tries, '>', 0, 'some pages moved back to global pool'); } $value = "B"x7000; for (1 .. $keycount) { print $sock "set ifoo$_ 0 0 7000 noreply\r\n$value\r\n"; } my $missing = 0; my $hits = 0; for (1 .. $keycount) { print $sock "get ifoo$_\r\n"; my $body = scalar(<$sock>); my $expected = "VALUE ifoo$_ 0 7000\r\n$value\r\nEND\r\n"; if ($body =~ /^END/) { $missing++; } else { $body .= scalar(<$sock>) . scalar(<$sock>); if ($body ne $expected) { print STDERR "Something terrible has happened: $expected\nBODY:\n$body\nDONETEST\n"; } else { $hits++; } } } #print STDERR "HITS: $hits, MISSES: $missing\n"; { my $stats = mem_stats($sock); cmp_ok($stats->{evictions}, '<', 2000, 'evictions were less than 2000'); # for ('evictions', 'reclaimed', 'curr_items', 'cmd_set', 'bytes') { # print STDERR "$_: ", $stats->{$_}, "\n"; # } } # Force reassign evictions by moving too much memory manually. { my $s = mem_stats($sock, 'slabs'); my $max_pages = 0; my $scls = 0; for my $k (keys %$s) { next unless $k =~ m/^(\d+)\:total_pages/; if ($s->{$k} > $max_pages) { $max_pages = $s->{$k}; $scls = $1; } } my $tries; for ($tries = 10; $tries > 0; $tries--) { print $sock "slabs reassign $scls 1\r\n"; my $res = <$sock>; sleep 1; my $s = mem_stats($sock); last if $s->{slab_reassign_evictions_nomem} > 0; } cmp_ok($tries, '>', 0, 'some reassign evictions happened'); } cmp_ok($hits, '>', 2000, 'were able to fetch back some of the small keys'); my $stats_done = mem_stats($sock); cmp_ok($stats_done->{slab_reassign_rescues}, '>', 0, 'some reassign rescues happened'); print $sock "flush_all\r\n"; is(scalar <$sock>, "OK\r\n", "did flush_all"); my $tries; for ($tries = 20; $tries > 0; $tries--) { sleep 1; my $stats = mem_stats($sock); if ($stats->{slab_global_page_pool} > 50) { last; } } cmp_ok($tries, '>', 0, 'reclaimed at least 50 pages before timeout'); { my $stats = mem_stats($sock, "slabs"); is($stats->{total_malloced}, 62914560, "total_malloced is what we expect"); } # Set into an entirely new class. Overload a bit to try to cause problems. $value = "B"x4096; for (1 .. $keycount * 4) { print $sock "set jfoo$_ 0 0 4096 noreply\r\n$value\r\n"; } { my $stats = mem_stats($sock); cmp_ok($stats->{curr_items}, '>', 10000, "stored at least 10000 4k items"); is($stats->{slab_global_page_pool}, 0, "drained the global page pool"); } { my $stats = mem_stats($sock, "slabs"); is($stats->{total_malloced}, 62914560, "total_malloced is same after re-assignment"); } memcached-1.5.22/t/multiversioning.t0000775000175000017500000000355013575022500014325 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 13; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $sock2 = $server->new_sock; ok($sock != $sock2, "have two different connections open"); # set large value my $size = 256 * 1024; # 256 kB my $bigval = "0123456789abcdef" x ($size / 16); $bigval =~ s/^0/\[/; $bigval =~ s/f$/\]/; my $bigval2 = uc($bigval); print $sock "set big 0 0 $size\r\n$bigval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "big", $bigval, "big value got correctly"); print $sock "get big\r\n"; my $buf; my $read = 0; my $to_read = $size / 2; while ($read < $to_read) { my $read_bytes = $sock->sysread($buf, ($to_read - $read > MemcachedTest::MAX_READ_WRITE_SIZE ? MemcachedTest::MAX_READ_WRITE_SIZE : $to_read - $read), $read); last if ($read_bytes <= 0); $read += $read_bytes; } is($read, $size / 2, "read half the answer back"); like($buf, qr/VALUE big/, "buf has big value header in it"); like($buf, qr/abcdef/, "buf has some data in it"); unlike($buf, qr/abcde\]/, "buf doesn't yet close"); # sock2 interrupts (maybe sock1 is slow) and deletes stuff: print $sock2 "delete big\r\n"; is(scalar <$sock2>, "DELETED\r\n", "deleted big from sock2 while sock1's still reading it"); mem_get_is($sock2, "big", undef, "nothing from sock2 now. gone from namespace."); print $sock2 "set big 0 0 $size\r\n$bigval2\r\n"; is(scalar <$sock2>, "STORED\r\n", "stored big w/ val2"); mem_get_is($sock2, "big", $bigval2, "big value2 got correctly"); # sock1 resumes reading... $buf .= <$sock>; $buf .= <$sock>; like($buf, qr/abcde\]/, "buf now closes"); # and if sock1 reads again, it's the uppercase version: mem_get_is($sock, "big", $bigval2, "big value2 got correctly from sock1"); memcached-1.5.22/t/lru-maintainer.t0000664000175000017500000000727013461170555014026 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More tests => 226; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; # Regression test for underestimating the size of items after the large memory # change. my $server = new_memcached('-m 3 -o lru_maintainer,lru_crawler -l 127.0.0.1'); my $sock = $server->sock; my $keystub = "X"x200; for (1 .. 15000) { print $sock "set $keystub$_ 0 0 2 noreply\r\nok\r\n"; } # There's probably already an error on the wire, so we'll see that. $keystub .= "20001"; print $sock "set $keystub 0 0 2\r\nok\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key without OOM"); # Basic tests $server = new_memcached('-m 6 -o lru_maintainer,lru_crawler -l 127.0.0.1'); $sock = $server->sock; for (1 .. 10) { print $sock "set ifoo$_ 0 1 2\r\nok\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); } sleep 3; { my $stats = mem_stats($sock); is($stats->{reclaimed}, 10, "expired key automatically reclaimed"); } my $value = "B"x66560; print $sock "set canary 0 0 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored canary key"); # Now flush the slab class with junk. for (my $key = 0; $key < 100; $key++) { if ($key == 30) { my $stats; for (0..2) { # Give the juggler some time to move. some platforms suffer at # this more than others (solaris amd64?) $stats = mem_stats($sock, "items"); if ($stats->{"items:31:moves_to_cold"} == 0) { sleep 1; next; } last; } isnt($stats->{"items:31:moves_to_cold"}, 0, "moved some items to cold"); # Items need two fetches to become active mem_get_is($sock, "canary", $value); mem_get_is($sock, "canary", $value); $stats = mem_stats($sock); # The maintainer thread needs to juggle a bit to actually rescue an # item. If it's slow we could evict after resuming setting. sleep 1; for (0..4) { my $s2 = mem_stats($sock); if ($s2->{lru_maintainer_juggles} - $stats->{lru_maintainer_juggles} < 5) { sleep 1; next; } last; } $stats = mem_stats($sock, "items"); isnt($stats->{"items:31:moves_to_warm"}, 0, "our canary moved to warm"); } print $sock "set key$key 0 0 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key$key"); } { my $stats = mem_stats($sock); isnt($stats->{evictions}, 0, "some evictions happened"); use Data::Dumper qw/Dumper/; } # Key should've been saved to the WARM_LRU, and still exists. mem_get_is($sock, "canary", $value); # Test TEMP_LRU $server = new_memcached('-m 2 -o lru_maintainer,lru_crawler,temporary_ttl=61'); $sock = $server->sock; { my $stats = mem_stats($sock, "settings"); is($stats->{temp_lru}, "yes"); } print $sock "set canary 0 30 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored temporary canary key"); { my $stats = mem_stats($sock, "items"); is($stats->{"items:31:number_hot"}, 0, "item did not go into hot LRU"); } # *Not* fetching the key, and flushing the slab class with junk. # Using keys with higher TTL's here. for (my $key = 0; $key < 100; $key++) { print $sock "set key$key 0 3600 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key$key"); } { my $stats = mem_stats($sock, "items"); isnt($stats->{"items:31:evictions"}, 0, "some evictions happened"); isnt($stats->{"items:31:number_hot"}, 0, "high exptime items went into hot LRU"); is($stats->{"items:31:number_temp"}, 1, "still one item in temporary LRU"); } # Canary should still exist, even unfetched, because it's protected by # temp LRU mem_get_is($sock, "canary", $value); memcached-1.5.22/t/authfile0000664000175000017500000000002313575022500012413 00000000000000foo:bar baaaz:quux memcached-1.5.22/t/sasl/0000775000175000017500000000000013261765410011724 500000000000000memcached-1.5.22/t/sasl/memcached.conf0000664000175000017500000000011713250303500014402 00000000000000mech_list: plain cram-md5 log_level: 5 sasldb_path: /tmp/test-memcached.sasldb memcached-1.5.22/t/issue_61.t0000664000175000017500000000141513615376577012545 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 7; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached("-R 1"); my $sock = $server->sock; print $sock "set foobar 0 0 5\r\nBubba\r\nset foobar 0 0 5\r\nBubba\r\nset foobar 0 0 5\r\nBubba\r\nset foobar 0 0 5\r\nBubba\r\nset foobar 0 0 5\r\nBubba\r\nset foobar 0 0 5\r\nBubba\r\n"; is (scalar <$sock>, "STORED\r\n", "stored foobar"); is (scalar <$sock>, "STORED\r\n", "stored foobar"); is (scalar <$sock>, "STORED\r\n", "stored foobar"); is (scalar <$sock>, "STORED\r\n", "stored foobar"); is (scalar <$sock>, "STORED\r\n", "stored foobar"); is (scalar <$sock>, "STORED\r\n", "stored foobar"); my $stats = mem_stats($sock); is ($stats->{"conn_yields"}, "5", "Got a decent number of yields"); memcached-1.5.22/t/stats.t0000775000175000017500000001274013615376577012253 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 108; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached("-o no_lru_crawler,no_lru_maintainer"); my $sock = $server->sock; ## Output looks like this: ## ## STAT pid 22969 ## STAT uptime 13 ## STAT time 1259170891 ## STAT version 1.4.3 ## STAT libevent 1.4.13-stable. ## see doc/protocol.txt for others # note that auth stats are tested in auth specific tests my $stats = mem_stats($sock); # Test number of keys if (MemcachedTest::enabled_tls_testing()) { # when TLS is enabled, stats contains time_since_server_cert_refresh is(scalar(keys(%$stats)), 72, "expected count of stats values"); } else { is(scalar(keys(%$stats)), 71, "expected count of stats values"); } # Test initial state foreach my $key (qw(curr_items total_items bytes cmd_get cmd_set get_hits evictions get_misses get_expired bytes_written delete_hits delete_misses incr_hits incr_misses decr_hits get_flushed decr_misses listen_disabled_num lrutail_reflocked time_in_listen_disabled_us)) { is($stats->{$key}, 0, "initial $key is zero"); } is($stats->{accepting_conns}, 1, "initial accepting_conns is one"); # Do some operations print $sock "set foo 0 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); my $stats = mem_stats($sock); foreach my $key (qw(total_items curr_items cmd_get cmd_set get_hits)) { is($stats->{$key}, 1, "after one set/one get $key is 1"); } my $cache_dump = mem_stats($sock, " cachedump 1 100"); ok(defined $cache_dump->{'foo'}, "got foo from cachedump"); print $sock "delete foo\r\n"; is(scalar <$sock>, "DELETED\r\n", "deleted foo"); my $stats = mem_stats($sock); is($stats->{delete_hits}, 1); is($stats->{delete_misses}, 0); print $sock "delete foo\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "shouldn't delete foo again"); my $stats = mem_stats($sock); is($stats->{delete_hits}, 1); is($stats->{delete_misses}, 1); # incr stats sub check_incr_stats { my ($ih, $im, $dh, $dm) = @_; my $stats = mem_stats($sock); is($stats->{incr_hits}, $ih); is($stats->{incr_misses}, $im); is($stats->{decr_hits}, $dh); is($stats->{decr_misses}, $dm); } print $sock "incr i 1\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "shouldn't incr a missing thing"); check_incr_stats(0, 1, 0, 0); print $sock "decr d 1\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "shouldn't decr a missing thing"); check_incr_stats(0, 1, 0, 1); print $sock "set n 0 0 1\r\n0\r\n"; is(scalar <$sock>, "STORED\r\n", "stored n"); print $sock "incr n 3\r\n"; is(scalar <$sock>, "3\r\n", "incr works"); check_incr_stats(1, 1, 0, 1); print $sock "decr n 1\r\n"; is(scalar <$sock>, "2\r\n", "decr works"); check_incr_stats(1, 1, 1, 1); # cas stats sub check_cas_stats { my ($ch, $cm, $cb) = @_; my $stats = mem_stats($sock); is($stats->{cas_hits}, $ch); is($stats->{cas_misses}, $cm); is($stats->{cas_badval}, $cb); } check_cas_stats(0, 0, 0); print $sock "cas c 0 0 1 99999999\r\nz\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "missed cas"); check_cas_stats(0, 1, 0); print $sock "set c 0 0 1\r\nx\r\n"; is(scalar <$sock>, "STORED\r\n", "stored c"); my ($id, $v) = mem_gets($sock, 'c'); is('x', $v, 'got the expected value'); print $sock "cas c 0 0 1 99999999\r\nz\r\n"; is(scalar <$sock>, "EXISTS\r\n", "missed cas"); check_cas_stats(0, 1, 1); my ($newid, $v) = mem_gets($sock, 'c'); is('x', $v, 'got the expected value'); print $sock "cas c 0 0 1 $id\r\nz\r\n"; is(scalar <$sock>, "STORED\r\n", "good cas"); check_cas_stats(1, 1, 1); my ($newid, $v) = mem_gets($sock, 'c'); is('z', $v, 'got the expected value'); my $settings = mem_stats($sock, ' settings'); is(1024, $settings->{'maxconns'}); # we run SSL tests over TCP; hence the domain_socket # is expected to be NULL. if (enabled_tls_testing()) { is('NULL', $settings->{'domain_socket'}); } else { isnt('NULL', $settings->{'domain_socket'}); } is('on', $settings->{'evictions'}); is('yes', $settings->{'cas_enabled'}); is('no', $settings->{'auth_enabled_sasl'}); print $sock "stats reset\r\n"; is(scalar <$sock>, "RESET\r\n", "good stats reset"); my $stats = mem_stats($sock); is(0, $stats->{'cmd_get'}); is(0, $stats->{'cmd_set'}); is(0, $stats->{'get_hits'}); is(0, $stats->{'get_misses'}); is(0, $stats->{'get_expired'}); is(0, $stats->{'get_flushed'}); is(0, $stats->{'delete_misses'}); is(0, $stats->{'delete_hits'}); is(0, $stats->{'incr_misses'}); is(0, $stats->{'incr_hits'}); is(0, $stats->{'decr_misses'}); is(0, $stats->{'decr_hits'}); is(0, $stats->{'cas_misses'}); is(0, $stats->{'cas_hits'}); is(0, $stats->{'cas_badval'}); is(0, $stats->{'evictions'}); is(0, $stats->{'reclaimed'}); is(0, $stats->{'lrutail_reflocked'}); # item expired print $sock "set should_expire 0 2678400 6\r\nfooval\r\n"; #2678400 = 31 days in seconds is(scalar <$sock>, "STORED\r\n", "set item to expire"); print $sock "get should_expire\r\n"; is(scalar <$sock>, "END\r\n", "item not returned"); my $stats = mem_stats($sock); is(1, $stats->{'get_expired'}, "get_expired counter is 1"); print $sock "set should_be_flushed 0 0 6\r\nbooval\r\n"; is(scalar <$sock>, "STORED\r\n", "set item to flush"); print $sock "flush_all\r\n"; is(scalar <$sock>, "OK\r\n", "flushed"); print $sock "get should_be_flushed\r\n"; is(scalar <$sock>, "END\r\n", "flushed item not returned"); my $stats = mem_stats($sock); is($stats->{cmd_flush}, 1, "after one flush cmd_flush is 1"); is($stats->{get_flushed}, 1, "after flush and a get, get_flushed is 1"); memcached-1.5.22/t/issue_68.t0000664000175000017500000000077113250303500012523 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 996; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; for (my $keyi = 1; $keyi < 250; $keyi++) { my $key = "x" x $keyi; print $sock "set $key 0 0 1\r\n9\r\n"; is (scalar <$sock>, "STORED\r\n", "stored $key"); mem_get_is($sock, $key, "9"); print $sock "incr $key 1\r\n"; is (scalar <$sock>, "10\r\n", "incr $key to 10"); mem_get_is($sock, $key, "10"); } memcached-1.5.22/t/expirations.t0000775000175000017500000000332313461170555013442 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 15; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $expire; sub wait_for_early_second { my $have_hires = eval "use Time::HiRes (); 1"; if ($have_hires) { my $tsh = Time::HiRes::time(); my $ts = int($tsh); return if ($tsh - $ts) < 0.5; } my $ts = int(time()); while (1) { my $t = int(time()); return if $t != $ts; select undef, undef, undef, 0.10; # 1/10th of a second sleeps until time changes. } } wait_for_early_second(); print $sock "set foo 0 3 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); sleep(4); mem_get_is($sock, "foo", undef); $expire = time() - 1; print $sock "set foo 0 $expire 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", undef, "already expired"); $expire = time() + 1; print $sock "set foo 0 $expire 6\r\nfoov+1\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "foov+1"); sleep(2.2); mem_get_is($sock, "foo", undef, "now expired"); $expire = time() - 20; print $sock "set boo 0 $expire 6\r\nbooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored boo"); mem_get_is($sock, "boo", undef, "now expired"); print $sock "add add 0 2 6\r\naddval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored add"); mem_get_is($sock, "add", "addval"); # second add fails print $sock "add add 0 2 7\r\naddval2\r\n"; is(scalar <$sock>, "NOT_STORED\r\n", "add failure"); sleep(2.3); print $sock "add add 0 2 7\r\naddval3\r\n"; is(scalar <$sock>, "STORED\r\n", "stored add again"); mem_get_is($sock, "add", "addval3"); memcached-1.5.22/t/issue_41.t0000664000175000017500000000172413250303500012511 00000000000000#!/usr/bin/perl use strict; use warnings; use POSIX qw(ceil); use Test::More tests => 691; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $factor = 2; my $val = "x" x $factor; my $key = ''; # SET items of diverse size to the daemon so it can attempt # to return a large stats output for slabs for (my $i=0; $i<69; $i++) { for (my $j=0; $j<10; $j++) { $key = "$i:$j"; print $sock "set key$key 0 0 $factor\r\n$val\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key$key"); } $factor *= 1.2; $factor = ceil($factor); $val = "x" x $factor; } # This request will kill the daemon if it has not allocated # enough memory internally. my $stats = mem_stats($sock, "slabs"); # Verify whether the daemon is still running or not by asking # it for statistics. print $sock "version\r\n"; my $v = scalar <$sock>; ok(defined $v && length($v), "memcached didn't respond"); memcached-1.5.22/t/refhang.t0000664000175000017500000000346613461170555012514 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; plan skip_all => 'Test is flaky. Needs special hooks.'; plan tests => 127; # start up a server with 10 maximum connections my $server = new_memcached("-m 6"); my $sock = $server->sock; my $hangsock = $server->new_sock; my $hangsock2 = $server->new_sock; my $value = "B"x66560; my $key = 0; # These aren't set to expire. my $mget = ''; for ($key = 0; $key < 120; $key++) { $mget .= "key$key " if $key < 115; print $sock "set key$key 0 0 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key$key"); } chop $mget; my $stats = mem_stats($sock, "items"); isnt($stats->{"items:31:evicted"}, "0", "check evicted"); my $lrutail_reflocked = $stats->{"items:31:lrutail_reflocked"}; is($lrutail_reflocked, "0", "check no slab lrutail_reflocked"); $stats = mem_stats($sock); is($stats->{"lrutail_reflocked"}, "0", "check no total lrutail_reflocked"); # Don't intend to read the results, need to fill the socket. # TODO: This test would be smarter if we cranked down the socket buffers # first? Or perhaps used a unix domain socket. print $hangsock "get $mget\r\n"; #sleep 8; # Now we try a bunch of sets again, and see if they start coming back as OOM's for ($key = 121; $key < 240; $key++) { print $sock "set key$key 0 0 66560\r\n$value\r\n"; my $res = scalar <$sock>; } $stats = mem_stats($sock, "items"); # Some items will OOM since we only clear a handful per alloc attempt. ok($stats->{"items:31:outofmemory"} > 0, "some ooms happened"); ok($stats->{"items:31:outofmemory"} < 20, "fewer than 20 ooms"); isnt($stats->{"items:31:lrutail_reflocked"}, "0", "nonzero lrutail_reflocked"); $stats = mem_stats($sock); isnt($stats->{"lrutail_reflocked"}, "0", "nonzero total lrutail_reflocked"); memcached-1.5.22/t/issue_3.t0000664000175000017500000000252413250303500012426 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 8; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $key = "del_key"; print $sock "delete $key\r\n"; is (scalar <$sock>, "NOT_FOUND\r\n", "not found on delete"); print $sock "delete $key 10\r\n"; is (scalar <$sock>, "CLIENT_ERROR bad command line format." . " Usage: delete [noreply]\r\n", "invalid delete"); print $sock "add $key 0 0 1\r\nx\r\n"; is (scalar <$sock>, "STORED\r\n", "Add before a broken delete."); print $sock "delete $key 10 noreply\r\n"; # Does not reply # is (scalar <$sock>, "ERROR\r\n", "Even more invalid delete"); print $sock "add $key 0 0 1\r\nx\r\n"; is (scalar <$sock>, "NOT_STORED\r\n", "Failed to add after failed silent delete."); print $sock "delete $key noreply\r\n"; # Will not reply, so let's do a set and check that. print $sock "set $key 0 0 1\r\nx\r\n"; is (scalar <$sock>, "STORED\r\n", "Stored a key"); print $sock "delete $key\r\n"; is (scalar <$sock>, "DELETED\r\n", "Properly deleted"); print $sock "set $key 0 0 1\r\nx\r\n"; is (scalar <$sock>, "STORED\r\n", "Stored a key"); print $sock "delete $key noreply\r\n"; # will not reply, but a subsequent add will succeed print $sock "add $key 0 0 1\r\nx\r\n"; is (scalar <$sock>, "STORED\r\n", "Add succeeded after deletion."); memcached-1.5.22/t/udp.t0000775000175000017500000002274513615376577011713 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 48; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use constant IS_ASCII => 0; use constant IS_BINARY => 1; use constant ENTRY_EXISTS => 0; use constant ENTRY_MISSING => 1; use constant BIN_REQ_MAGIC => 0x80; use constant BIN_RES_MAGIC => 0x81; use constant CMD_GET => 0x00; use constant CMD_SET => 0x01; use constant CMD_ADD => 0x02; use constant CMD_REPLACE => 0x03; use constant CMD_DELETE => 0x04; use constant CMD_INCR => 0x05; use constant CMD_DECR => 0x06; use constant CMD_APPEND => 0x0E; use constant CMD_PREPEND => 0x0F; use constant REQ_PKT_FMT => "CCnCCnNNNN"; use constant RES_PKT_FMT => "CCnCCnNNNN"; use constant INCRDECR_PKT_FMT => "NNNNN"; use constant MIN_RECV_BYTES => length(pack(RES_PKT_FMT)); my $server = new_memcached("-l 127.0.0.1"); my $sock = $server->sock; # set foo (and should get it) print $sock "set foo 0 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); my $usock = $server->new_udp_sock or die "Can't bind : $@\n"; # testing sequence of request ids for my $offt (1, 1, 2) { my $req = 160 + $offt; my $res = send_udp_request($usock, $req, "get foo\r\n"); ok($res, "got result"); is(keys %$res, 1, "one key (one packet)"); ok($res->{0}, "only got seq number 0"); is(substr($res->{0}, 8), "VALUE foo 0 6\r\nfooval\r\nEND\r\n"); is(hexify(substr($res->{0}, 0, 2)), hexify(pack("n", $req)), "udp request number in response ($req) is correct"); } # op tests for my $prot (::IS_ASCII,::IS_BINARY) { udp_set_test($prot,45,"aval$prot","1",0,0); udp_set_test($prot,45,"bval$prot","abcd" x 1024,0,0); udp_get_test($prot,45,"aval$prot","1",::ENTRY_EXISTS); udp_get_test($prot,45,"404$prot","1",::ENTRY_MISSING); udp_incr_decr_test($prot,45,"aval$prot","1","incr",1); udp_incr_decr_test($prot,45,"aval$prot","1","decr",2); udp_delete_test($prot,45,"aval$prot"); } sub udp_set_test { my ($protocol, $req_id, $key, $value, $flags, $exp) = @_; my $req = ""; my $val_len = length($value); if ($protocol == ::IS_ASCII) { $req = "set $key $flags $exp $val_len\r\n$value\r\n"; } elsif ($protocol == ::IS_BINARY) { my $key_len = length($key); my $extra = pack "NN",$flags,$exp; my $extra_len = length($extra); my $total_len = $val_len + $extra_len + $key_len; $req = pack(::REQ_PKT_FMT, ::BIN_REQ_MAGIC, ::CMD_SET, $key_len, $extra_len, 0, 0, $total_len, 0, 0, 0); $req .= $extra . $key . $value; } my $datagrams = send_udp_request($usock, $req_id, $req); my $resp = construct_udp_message($datagrams); if ($protocol == ::IS_ASCII) { is($resp,"STORED\r\n","Store key $key using ASCII protocol"); } elsif ($protocol == ::IS_BINARY) { my ($resp_magic, $resp_op_code, $resp_key_len, $resp_extra_len, $resp_data_type, $resp_status, $resp_total_len, $resp_opaque, $resp_ident_hi, $resp_ident_lo) = unpack(::RES_PKT_FMT, $resp); is($resp_status,"0","Store key $key using binary protocol"); } } sub udp_get_test { my ($protocol, $req_id, $key, $value, $exists) = @_; my $key_len = length($key); my $value_len = length($value); my $req = ""; if ($protocol == ::IS_ASCII) { $req = "get $key\r\n"; } elsif ($protocol == ::IS_BINARY) { $req = pack(::REQ_PKT_FMT, ::BIN_REQ_MAGIC, ::CMD_GET, $key_len, 0, 0, 0, $key_len, 0, 0, 0); $req .= $key; } my $datagrams = send_udp_request($usock, $req_id, $req); my $resp = construct_udp_message($datagrams); if ($protocol == ::IS_ASCII) { if ($exists == ::ENTRY_EXISTS) { is($resp,"VALUE $key 0 $value_len\r\n$value\r\nEND\r\n","Retrieve entry with key $key using ASCII protocol"); } else { is($resp,"END\r\n","Retrieve non existing entry with key $key using ASCII protocol"); } } elsif ($protocol == ::IS_BINARY) { my ($resp_magic, $resp_op_code, $resp_key_len, $resp_extra_len, $resp_data_type, $resp_status, $resp_total_len, $resp_opaque, $resp_ident_hi, $resp_ident_lo) = unpack(::RES_PKT_FMT, $resp); if ($exists == ::ENTRY_EXISTS) { is($resp_status,"0","Retrieve entry with key $key using binary protocol"); is(substr($resp,::MIN_RECV_BYTES + $resp_extra_len + $resp_key_len, $value_len),$value,"Value for key $key retrieved with binary protocol matches"); } else { is($resp_status,"1","Retrieve non existing entry with key $key using binary protocol"); } } } sub udp_delete_test { my ($protocol, $req_id, $key) = @_; my $req = ""; my $key_len = length($key); if ($protocol == ::IS_ASCII) { $req = "delete $key\r\n"; } elsif ($protocol == ::IS_BINARY) { $req = pack(::REQ_PKT_FMT, ::BIN_REQ_MAGIC, ::CMD_DELETE, $key_len, 0, 0, 0, $key_len, 0, 0, 0); $req .= $key; } my $datagrams = send_udp_request($usock, $req_id, $req); my $resp = construct_udp_message($datagrams); if ($protocol == ::IS_ASCII) { is($resp,"DELETED\r\n","Delete key $key using ASCII protocol"); } elsif ($protocol == ::IS_BINARY) { my ($resp_magic, $resp_op_code, $resp_key_len, $resp_extra_len, $resp_data_type, $resp_status, $resp_total_len, $resp_opaque, $resp_ident_hi, $resp_ident_lo) = unpack(::RES_PKT_FMT, $resp); is($resp_status,"0","Delete key $key using binary protocol"); } } sub udp_incr_decr_test { my ($protocol, $req_id, $key, $val, $optype, $init_val) = @_; my $req = ""; my $key_len = length($key); my $expected_value = 0; my $acmd = "incr"; my $bcmd = ::CMD_INCR; if ($optype eq "incr") { $expected_value = $init_val + $val; } else { $acmd = "decr"; $bcmd = ::CMD_DECR; $expected_value = $init_val - $val; } if ($protocol == ::IS_ASCII) { $req = "$acmd $key $val\r\n"; } elsif ($protocol == ::IS_BINARY) { my $extra = pack(::INCRDECR_PKT_FMT, ($val / 2 ** 32),($val % 2 ** 32), 0, 0, 0); my $extra_len = length($extra); $req = pack(::REQ_PKT_FMT, ::BIN_REQ_MAGIC, $bcmd, $key_len, $extra_len, 0, 0, $key_len + $extra_len, 0, 0, 0); $req .= $extra . $key; } my $datagrams = send_udp_request($usock, $req_id, $req); my $resp = construct_udp_message($datagrams); if ($protocol == ::IS_ASCII) { is($resp,"$expected_value\r\n","perform $acmd math operation on key $key with ASCII protocol"); } elsif ($protocol == ::IS_BINARY) { my ($resp_magic, $resp_op_code, $resp_key_len, $resp_extra_len, $resp_data_type, $resp_status, $resp_total_len, $resp_opaque, $resp_ident_hi, $resp_ident_lo) = unpack(::RES_PKT_FMT, $resp); is($resp_status,"0","perform $acmd math operation on key $key with binary protocol"); my ($resp_hi,$resp_lo) = unpack("NN",substr($resp,::MIN_RECV_BYTES + $resp_extra_len + $resp_key_len, $resp_total_len - $resp_extra_len - $resp_key_len)); is(($resp_hi * 2 ** 32) + $resp_lo,$expected_value,"validate result of binary protocol math operation $acmd . Expected value $expected_value") } } sub construct_udp_message { my $datagrams = shift; my $num_datagram = keys (%$datagrams); my $msg = ""; my $cur_dg =""; my $cur_udp_header =""; for (my $cur_dg_index = 0; $cur_dg_index < $num_datagram; $cur_dg_index++) { $cur_dg = $datagrams->{$cur_dg_index}; isnt($cur_dg,"","missing datagram for segment $cur_dg_index"); $cur_udp_header=substr($cur_dg, 0, 8); $msg .= substr($cur_dg,8); } return $msg; } sub hexify { my $val = shift; $val =~ s/(.)/sprintf("%02x", ord($1))/egs; return $val; } # returns undef on select timeout, or hashref of "seqnum" -> payload (including headers) # verifies that resp_id is equal to id sent in request # ensures consistency in num packets that make up response sub send_udp_request { my ($sock, $reqid, $req) = @_; my $pkt = pack("nnnn", $reqid, 0, 1, 0); # request id (opaque), seq num, #packets, reserved (must be 0) $pkt .= $req; my $fail = sub { my $msg = shift; warn " FAILING send_udp because: $msg\n"; return undef; }; return $fail->("send") unless send($sock, $pkt, 0); my $ret = {}; my $got = 0; # packets got my $numpkts = undef; while (!defined($numpkts) || $got < $numpkts) { my $rin = ''; vec($rin, fileno($sock), 1) = 1; my $rout; return $fail->("timeout after $got packets") unless select($rout = $rin, undef, undef, 1.5); my $res; my $sender = $sock->recv($res, 1500, 0); my ($resid, $seq, $this_numpkts, $resv) = unpack("nnnn", substr($res, 0, 8)); die "Response ID of $resid doesn't match request if of $reqid" unless $resid == $reqid; die "Reserved area not zero" unless $resv == 0; die "num packets changed midstream!" if defined $numpkts && $this_numpkts != $numpkts; $numpkts = $this_numpkts; $ret->{$seq} = $res; $got++; } return $ret; } __END__ $sender = recv($usock, $ans, 1050, 0); __END__ $usock->send ($hispaddr = recv(SOCKET, $rtime, 4, 0)) || die "recv: $!"; ($port, $hisiaddr) = sockaddr_in($hispaddr); $host = gethostbyaddr($hisiaddr, AF_INET); $histime = unpack("N", $rtime) - $SECS_of_70_YEARS ; memcached-1.5.22/t/flush-all.t0000775000175000017500000000577213461170555012776 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 26; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $expire; print $sock "set foo 0 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); print $sock "flush_all\r\n"; is(scalar <$sock>, "OK\r\n", "did flush_all"); mem_get_is($sock, "foo", undef); # Test flush_all with zero delay. print $sock "set foo 0 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); print $sock "flush_all 0\r\n"; is(scalar <$sock>, "OK\r\n", "did flush_all"); mem_get_is($sock, "foo", undef); # check that flush_all doesn't blow away items that immediately get set print $sock "set foo 0 0 3\r\nnew\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo = 'new'"); mem_get_is($sock, "foo", 'new'); # and the other form, specifying a flush_all time... my $expire = time() + 2; print $sock "flush_all $expire\r\n"; is(scalar <$sock>, "OK\r\n", "did flush_all in future"); print $sock "set foo 0 0 4\r\n1234\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo = '1234'"); mem_get_is($sock, "foo", '1234'); sleep(5); mem_get_is($sock, "foo", undef); print $sock "set foo 0 0 5\r\n12345\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo = '12345'"); mem_get_is($sock, "foo", '12345'); print $sock "flush_all 86400\r\n"; is(scalar <$sock>, "OK\r\n", "did flush_all for far future"); # Check foo still exists. mem_get_is($sock, "foo", '12345'); print $sock "set foo2 0 0 5\r\n54321\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo2 = '54321'"); mem_get_is($sock, "foo", '12345'); mem_get_is($sock, "foo2", '54321'); # Test -F option which disables flush_all $server = new_memcached('-F'); $sock = $server->sock; print $sock "set foo 0 0 7\r\nfooval2\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval2"); print $sock "flush_all\r\n"; is(scalar <$sock>, "CLIENT_ERROR flush_all not allowed\r\n", "flush_all was not allowed"); mem_get_is($sock, "foo", "fooval2"); # Test that disabling CAS makes flush_all less accurate. # Due to lock ordering issues we can no longer evict items newer than # oldest_live, so we rely on the CAS counter for an exact cliff. So disabling # CAS now means all items set in the same second will fail to set. $server = new_memcached('-C'); $sock = $server->sock; my $failed_nocas = 0; # Running this 100,000 times failed the test a handful of times. 50 tries # should be enough. for (1..50) { print $sock "flush_all 0\r\n"; my $foo = scalar <$sock>; print $sock "set foo 0 0 3\r\nnew\r\n"; $foo = scalar <$sock>; print $sock "get foo\r\n"; my $line = scalar <$sock>; if ($line =~ /^VALUE/) { $line = scalar <$sock>; $line = scalar <$sock>; print STDERR "Succeeded!!!\n"; next; } elsif ($line =~ /^END/) { $failed_nocas++; last; } } is($failed_nocas, 1, "failed to set value after flush with no CAS at least once"); memcached-1.5.22/t/ssl_settings.t0000664000175000017500000000160413575022500013603 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use Cwd; if (!enabled_tls_testing()) { plan skip_all => 'SSL testing is not enabled'; exit 0; } my $server = new_memcached(); my $settings = mem_stats($server->sock, ' settings'); my $cert = getcwd ."/t/". MemcachedTest::SRV_CRT; my $key = getcwd ."/t/". MemcachedTest::SRV_KEY; is($settings->{'ssl_enabled'}, 'yes'); is($settings->{'ssl_chain_cert'}, $cert); is($settings->{'ssl_key'}, $key); is($settings->{'ssl_verify_mode'}, 0); is($settings->{'ssl_keyformat'}, 1); is($settings->{'ssl_ciphers'}, 'NULL'); is($settings->{'ssl_ca_cert'}, 'NULL'); is($settings->{'ssl_wbuf_size'}, 16384); $server->DESTROY(); $server = new_memcached("-o ssl_wbuf_size=64"); $settings = mem_stats($server->sock, ' settings'); is($settings->{'ssl_wbuf_size'},65536); done_testing(); memcached-1.5.22/t/extstore.t0000664000175000017500000001343413461170555012753 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use Data::Dumper qw/Dumper/; my $ext_path; if (!supports_extstore()) { plan skip_all => 'extstore not enabled'; exit 0; } $ext_path = "/tmp/extstore.$$"; my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,slab_automove=0,ext_compact_under=1"); my $sock = $server->sock; # Wait until all items have flushed sub wait_for_ext { my $target = shift || 0; my $sum = $target + 1; while ($sum > $target) { my $s = mem_stats($sock, "items"); $sum = 0; for my $key (keys %$s) { if ($key =~ m/items:(\d+):number/) { # Ignore classes which can contain extstore items next if $1 < 3; $sum += $s->{$key}; } } sleep 1 if $sum > $target; } } my $value; { my @chars = ("C".."Z"); for (1 .. 20000) { $value .= $chars[rand @chars]; } } # fill a small object print $sock "set foo 0 0 2\r\nhi\r\n"; is(scalar <$sock>, "STORED\r\n", "stored small value"); # fetch mem_get_is($sock, "foo", "hi"); # check extstore counters { my $stats = mem_stats($sock); is($stats->{extstore_objects_written}, 0); } # fill some larger objects { # set one canary value for later print $sock "set canary 0 0 20000 noreply\r\n$value\r\n"; my $keycount = 1000; for (1 .. $keycount) { print $sock "set nfoo$_ 0 0 20000 noreply\r\n$value\r\n"; } # wait for a flush wait_for_ext(); # fetch # TODO: Fetch back all values mem_get_is($sock, "nfoo1", $value); # check extstore counters my $stats = mem_stats($sock); cmp_ok($stats->{extstore_page_allocs}, '>', 0, 'at least one page allocated'); cmp_ok($stats->{extstore_objects_written}, '>', $keycount / 2, 'some objects written'); cmp_ok($stats->{extstore_bytes_written}, '>', length($value) * 2, 'some bytes written'); cmp_ok($stats->{get_extstore}, '>', 0, 'one object was fetched'); cmp_ok($stats->{extstore_objects_read}, '>', 0, 'one object read'); cmp_ok($stats->{extstore_bytes_read}, '>', length($value), 'some bytes read'); # Remove half of the keys for the next test. for (1 .. $keycount) { next unless $_ % 2 == 0; print $sock "delete nfoo$_ noreply\r\n"; } my $stats2 = mem_stats($sock); cmp_ok($stats->{extstore_bytes_used}, '>', $stats2->{extstore_bytes_used}, 'bytes used dropped after deletions'); cmp_ok($stats->{extstore_objects_used}, '>', $stats2->{extstore_objects_used}, 'objects used dropped after deletions'); is($stats2->{badcrc_from_extstore}, 0, 'CRC checks successful'); is($stats2->{miss_from_extstore}, 0, 'no misses'); # delete the rest for (1 .. $keycount) { next unless $_ % 2 == 1; print $sock "delete nfoo$_ noreply\r\n"; } } # fill to eviction { my $keycount = 4000; for (1 .. $keycount) { print $sock "set mfoo$_ 0 0 20000 noreply\r\n$value\r\n"; # wait to avoid evictions wait_for_ext(500) if ($_ % 2000 == 0); } # because item_age is set to 2s wait_for_ext(); my $stats = mem_stats($sock); is($stats->{evictions}, 0, 'no evictions'); is($stats->{miss_from_extstore}, 0, 'no misses'); # FIXME: test is flaky; something can rescue the canary because of a race # condition. might need to roundtrip twice or disable compaction? #mem_get_is($sock, "canary", undef); # check counters $stats = mem_stats($sock); cmp_ok($stats->{extstore_page_evictions}, '>', 0, 'at least one page evicted'); cmp_ok($stats->{extstore_objects_evicted}, '>', 0, 'at least one object evicted'); cmp_ok($stats->{extstore_bytes_evicted}, '>', 0, 'some bytes evicted'); cmp_ok($stats->{extstore_pages_free}, '<', 2, 'few pages are free'); #is($stats->{miss_from_extstore}, 1, 'exactly one miss'); # refresh some keys so rescues happen while drop_unread == 1. for (1 .. $keycount / 2) { next unless $_ % 2 == 1; # Need to be fetched twice in order to bump print $sock "touch mfoo$_ 0 noreply\r\n"; print $sock "touch mfoo$_ 0 noreply\r\n"; } print $sock "extstore drop_unread 1\r\n"; my $res = <$sock>; print $sock "extstore max_frag 0\r\n"; $res = <$sock>; print $sock "extstore compact_under 4\r\n"; $res = <$sock>; print $sock "extstore drop_under 3\r\n"; $res = <$sock>; for (1 .. $keycount) { next unless $_ % 2 == 0; print $sock "delete mfoo$_ noreply\r\n"; } sleep 4; $stats = mem_stats($sock); cmp_ok($stats->{extstore_pages_free}, '>', 0, 'some pages now free'); cmp_ok($stats->{extstore_compact_rescues}, '>', 0, 'some compaction rescues happened'); cmp_ok($stats->{extstore_compact_skipped}, '>', 0, 'some compaction skips happened'); print $sock "extstore drop_unread 0\r\n"; $res = <$sock>; } # attempt to incr/decr/append/prepend or chunk objects that were sent to disk. { my $keycount = 100; for (1 .. $keycount) { print $sock "set bfoo$_ 0 0 20000 noreply\r\n$value\r\n"; } wait_for_ext(); # incr should be blocked. print $sock "incr bfoo1 1\r\n"; is(scalar <$sock>, "CLIENT_ERROR cannot increment or decrement non-numeric value\r\n", 'incr fails'); # append/prepend *could* work, but it would require pulling the item back in. print $sock "append bfoo1 0 0 2\r\nhi\r\n"; is(scalar <$sock>, "NOT_STORED\r\n", 'append fails'); print $sock "prepend bfoo1 0 0 2\r\nhi\r\n"; is(scalar <$sock>, "NOT_STORED\r\n", 'prepend fails'); } done_testing(); END { unlink $ext_path if $ext_path; } memcached-1.5.22/t/quit.t0000664000175000017500000000112113461170555012046 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 1; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; { print $sock "quit\r\n"; # Five seconds ought to be enough to get hung up on. my $oldalarmt = alarm(5); # Verify we can't read anything. my $bytesread = -1; eval { local $SIG{'ALRM'} = sub { die "timeout" }; my $data = ""; $bytesread = sysread($sock, $data, 24), }; is($bytesread, 0, "Read after quit."); # Restore signal stuff. alarm($oldalarmt); } memcached-1.5.22/t/lib/0000755000175000017500000000000013611703161011520 500000000000000memcached-1.5.22/t/lib/MemcachedTest.pm0000664000175000017500000002756313611703161014523 00000000000000package MemcachedTest; use strict; use IO::Socket::INET; use IO::Socket::UNIX; use Exporter 'import'; use Carp qw(croak); use vars qw(@EXPORT); # Instead of doing the substitution with Autoconf, we assume that # cwd == builddir. use Cwd; my $builddir = getcwd; my @unixsockets = (); @EXPORT = qw(new_memcached sleep mem_get_is mem_gets mem_gets_is mem_stats supports_sasl free_port supports_drop_priv supports_extstore wait_ext_flush supports_tls enabled_tls_testing run_help); use constant MAX_READ_WRITE_SIZE => 16384; use constant SRV_CRT => "server_crt.pem"; use constant SRV_KEY => "server_key.pem"; use constant CLIENT_CRT => "client_crt.pem"; use constant CLIENT_KEY => "client_key.pem"; use constant CA_CRT => "cacert.pem"; my $testdir = $builddir . "/t/"; my $client_crt = $testdir. CLIENT_CRT; my $client_key = $testdir. CLIENT_KEY; my $server_crt = $testdir . SRV_CRT; my $server_key = $testdir . SRV_KEY; my $tls_checked = 0; sub sleep { my $n = shift; select undef, undef, undef, $n; } # Wait until all items have flushed sub wait_ext_flush { my $sock = shift; my $target = shift || 0; my $sum = $target + 1; while ($sum > $target) { my $s = mem_stats($sock, "items"); $sum = 0; for my $key (keys %$s) { if ($key =~ m/items:(\d+):number/) { # Ignore classes which can contain extstore items next if $1 < 3; $sum += $s->{$key}; } } sleep 1 if $sum > $target; } } sub mem_stats { my ($sock, $type) = @_; $type = $type ? " $type" : ""; print $sock "stats$type\r\n"; my $stats = {}; while (<$sock>) { last if /^(\.|END)/; /^(STAT|ITEM) (\S+)\s+([^\r\n]+)/; #print " slabs: $_"; $stats->{$2} = $3; } return $stats; } sub mem_get_is { # works on single-line values only. no newlines in value. my ($sock_opts, $key, $val, $msg) = @_; my $opts = ref $sock_opts eq "HASH" ? $sock_opts : {}; my $sock = ref $sock_opts eq "HASH" ? $opts->{sock} : $sock_opts; my $expect_flags = $opts->{flags} || 0; my $dval = defined $val ? "'$val'" : ""; $msg ||= "$key == $dval"; print $sock "get $key\r\n"; if (! defined $val) { my $line = scalar <$sock>; if ($line =~ /^VALUE/) { $line .= scalar(<$sock>) . scalar(<$sock>); } Test::More::is($line, "END\r\n", $msg); } else { my $len = length($val); my $body = scalar(<$sock>); my $expected = "VALUE $key $expect_flags $len\r\n$val\r\nEND\r\n"; if (!$body || $body =~ /^END/) { Test::More::is($body, $expected, $msg); return; } $body .= scalar(<$sock>) . scalar(<$sock>); Test::More::is($body, $expected, $msg); } } sub mem_gets { # works on single-line values only. no newlines in value. my ($sock_opts, $key) = @_; my $opts = ref $sock_opts eq "HASH" ? $sock_opts : {}; my $sock = ref $sock_opts eq "HASH" ? $opts->{sock} : $sock_opts; my $val; my $expect_flags = $opts->{flags} || 0; print $sock "gets $key\r\n"; my $response = <$sock>; if ($response =~ /^END/) { return "NOT_FOUND"; } else { $response =~ /VALUE (.*) (\d+) (\d+) (\d+)/; my $flags = $2; my $len = $3; my $identifier = $4; read $sock, $val , $len; # get the END $_ = <$sock>; $_ = <$sock>; return ($identifier,$val); } } sub mem_gets_is { # works on single-line values only. no newlines in value. my ($sock_opts, $identifier, $key, $val, $msg) = @_; my $opts = ref $sock_opts eq "HASH" ? $sock_opts : {}; my $sock = ref $sock_opts eq "HASH" ? $opts->{sock} : $sock_opts; my $expect_flags = $opts->{flags} || 0; my $dval = defined $val ? "'$val'" : ""; $msg ||= "$key == $dval"; print $sock "gets $key\r\n"; if (! defined $val) { my $line = scalar <$sock>; if ($line =~ /^VALUE/) { $line .= scalar(<$sock>) . scalar(<$sock>); } Test::More::is($line, "END\r\n", $msg); } else { my $len = length($val); my $body = scalar(<$sock>); my $expected = "VALUE $key $expect_flags $len $identifier\r\n$val\r\nEND\r\n"; if (!$body || $body =~ /^END/) { Test::More::is($body, $expected, $msg); return; } $body .= scalar(<$sock>) . scalar(<$sock>); Test::More::is($body, $expected, $msg); } } sub free_port { my $type = shift || "tcp"; my $sock; my $port; while (!$sock) { $port = int(rand(20000)) + 30000; if (enabled_tls_testing()) { $sock = eval qq{ IO::Socket::SSL->new(LocalAddr => '127.0.0.1', LocalPort => $port, Proto => '$type', ReuseAddr => 1, SSL_verify_mode => SSL_VERIFY_NONE); }; die $@ if $@; # sanity check. } else { $sock = IO::Socket::INET->new(LocalAddr => '127.0.0.1', LocalPort => $port, Proto => $type, ReuseAddr => 1); } } return $port; } sub supports_udp { my $output = `$builddir/memcached-debug -h`; return 0 if $output =~ /^memcached 1\.1\./; return 1; } sub supports_sasl { my $output = `$builddir/memcached-debug -h`; return 1 if $output =~ /sasl/i; return 0; } sub supports_extstore { my $output = `$builddir/memcached-debug -h`; return 1 if $output =~ /ext_path/i; return 0; } sub supports_tls { my $output = `$builddir/memcached-debug -h`; return 1 if $output =~ /enable-ssl/i; return 0; } sub enabled_tls_testing { if ($tls_checked) { return 1; } elsif (supports_tls() && $ENV{SSL_TEST}) { eval "use IO::Socket::SSL"; croak("IO::Socket::SSL not installed or failed to load, cannot run SSL tests as requested") if $@; $tls_checked = 1; return 1; } } sub supports_drop_priv { my $output = `$builddir/memcached-debug -h`; return 1 if $output =~ /no_drop_privileges/i; return 0; } sub get_memcached_exe { my $exe = "$builddir/memcached-debug"; croak("memcached binary doesn't exist. Haven't run 'make' ?\n") unless -e $exe; croak("memcached binary not executable\n") unless -x _; return $exe; } sub run_help { my $exe = get_memcached_exe(); return system("$exe -h"); } sub new_memcached { my ($args, $passed_port) = @_; my $port = $passed_port; my $host = '127.0.0.1'; my $ssl_enabled = enabled_tls_testing(); if ($ENV{T_MEMD_USE_DAEMON}) { my ($host, $port) = ($ENV{T_MEMD_USE_DAEMON} =~ m/^([^:]+):(\d+)$/); my $conn; if ($ssl_enabled) { $conn = eval qq{IO::Socket::SSL->new(PeerAddr => "$host:$port", SSL_verify_mode => SSL_VERIFY_NONE, SSL_cert_file => '$client_crt', SSL_key_file => '$client_key'); }; die $@ if $@; # sanity check. } else { $conn = IO::Socket::INET->new(PeerAddr => "$host:$port"); } if ($conn) { return Memcached::Handle->new(conn => $conn, host => $host, port => $port); } croak("Failed to connect to specified memcached server.") unless $conn; } if ($< == 0) { $args .= " -u root"; } $args .= " -o relaxed_privileges"; my $udpport; if ($args =~ /-l (\S+)/ || ($ssl_enabled && ($args !~ /-s (\S+)/))) { if (!$port) { $port = free_port(); } $udpport = free_port("udp"); $args .= " -p $port"; if (supports_udp() && $args !~ /-U (\S+)/) { $args .= " -U $udpport"; } if ($ssl_enabled) { $args .= " -Z -o ssl_chain_cert=$server_crt -o ssl_key=$server_key"; } } elsif ($args !~ /-s (\S+)/) { my $num = @unixsockets; my $file = "/tmp/memcachetest.$$.$num"; $args .= " -s $file"; push(@unixsockets, $file); } my $childpid = fork(); my $exe = get_memcached_exe(); unless ($childpid) { #print STDERR "RUN: $exe $args\n"; exec "$builddir/timedrun 600 $exe $args"; exit; # never gets here. } # unix domain sockets if ($args =~ /-s (\S+)/) { sleep 1; my $filename = $1; my $conn = IO::Socket::UNIX->new(Peer => $filename) || croak("Failed to connect to unix domain socket: $! '$filename'"); return Memcached::Handle->new(pid => $childpid, conn => $conn, domainsocket => $filename, host => $host, port => $port); } # try to connect / find open port, only if we're not using unix domain # sockets for (1..20) { my $conn; if ($ssl_enabled) { $conn = eval qq{ IO::Socket::SSL->new(PeerAddr => "127.0.0.1:$port", SSL_verify_mode => SSL_VERIFY_NONE, SSL_cert_file => '$client_crt', SSL_key_file => '$client_key'); }; die $@ if $@; # sanity check. } else { $conn = IO::Socket::INET->new(PeerAddr => "127.0.0.1:$port"); } if ($conn) { return Memcached::Handle->new(pid => $childpid, conn => $conn, udpport => $udpport, host => $host, port => $port); } select undef, undef, undef, 0.10; } croak("Failed to startup/connect to memcached server."); } END { for (@unixsockets) { unlink $_; } } ############################################################################ package Memcached::Handle; sub new { my ($class, %params) = @_; return bless \%params, $class; } sub DESTROY { my $self = shift; kill 2, $self->{pid}; } sub stop { my $self = shift; kill 15, $self->{pid}; } sub graceful_stop { my $self = shift; kill 'SIGUSR1', $self->{pid}; } sub host { $_[0]{host} } sub port { $_[0]{port} } sub udpport { $_[0]{udpport} } sub sock { my $self = shift; if ($self->{conn} && ($self->{domainsocket} || getpeername($self->{conn}))) { return $self->{conn}; } return $self->new_sock; } sub new_sock { my $self = shift; if ($self->{domainsocket}) { return IO::Socket::UNIX->new(Peer => $self->{domainsocket}); } elsif (MemcachedTest::enabled_tls_testing()) { return eval qq{ IO::Socket::SSL->new(PeerAddr => "$self->{host}:$self->{port}", SSL_verify_mode => IO::Socket::SSL::SSL_VERIFY_NONE, SSL_cert_file => '$client_crt', SSL_key_file => '$client_key'); }; } else { return IO::Socket::INET->new(PeerAddr => "$self->{host}:$self->{port}"); } } sub new_udp_sock { my $self = shift; return IO::Socket::INET->new(PeerAddr => '127.0.0.1', PeerPort => $self->{udpport}, Proto => 'udp', LocalAddr => '127.0.0.1', LocalPort => MemcachedTest::free_port('udp'), ); } 1; memcached-1.5.22/t/getandtouch.t0000664000175000017500000000251313461170555013377 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More tests => 15; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; # cache miss print $sock "gat 10 foo1\r\n"; is(scalar <$sock>, "END\r\n", "cache miss"); # set foo1 and foo2 (and should get it) print $sock "set foo1 0 2 7\r\nfooval1\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); print $sock "set foo2 0 2 7\r\nfooval2\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo2"); # get and touch it with cas print $sock "gats 10 foo1 foo2\r\n"; ok(scalar <$sock> =~ /VALUE foo1 0 7 (\d+)\r\n/, "get and touch foo1 with cas regexp success"); is(scalar <$sock>, "fooval1\r\n","value"); ok(scalar <$sock> =~ /VALUE foo2 0 7 (\d+)\r\n/, "get and touch foo2 with cas regexp success"); is(scalar <$sock>, "fooval2\r\n","value"); is(scalar <$sock>, "END\r\n", "end"); # get and touch it without cas print $sock "gat 10 foo1 foo2\r\n"; ok(scalar <$sock> =~ /VALUE foo1 0 7\r\n/, "get and touch foo1 without cas regexp success"); is(scalar <$sock>, "fooval1\r\n","value"); ok(scalar <$sock> =~ /VALUE foo2 0 7\r\n/, "get and touch foo2 without cas regexp success"); is(scalar <$sock>, "fooval2\r\n","value"); is(scalar <$sock>, "END\r\n", "end"); sleep 2; mem_get_is($sock, "foo1", "fooval1"); mem_get_is($sock, "foo2", "fooval2"); memcached-1.5.22/t/server_key.pem0000664000175000017500000000156713611710715013571 00000000000000-----BEGIN RSA PRIVATE KEY----- MIICXQIBAAKBgQDKdjV5keii7u/0NX4phXWQZVpg+KDZ72jRYXlp1+J78WdxZVAx Tpr4byfpBQsOdpUkm8K/kOVrRf3jVKzVYpBON96Prpb2s1frrbhEE12nNHbBJkmR Zz5eUmjBHH+Rx54B4r6nqOs/RG3HxYJLl9k/wFGZGyDfEqPmu2ZFbbThBwIDAQAB AoGBAItN/ItSVrRNHTN13wHovzSEWERiriJl9UQkAmtOTJqlRlyiriPPBxgrO1W8 z5e7BfGzbrNqmkBOX1uctnL4J3tA7xFO6OnquDCLXhLc49mw+zxkcDP5ta1ROeiV kIm8rSsvh8ks7StA17m5910rNfn5/IRHK6dC+G4FK7qLfCiRAkEA5Szy22sFeE4D OM/wmCHpoOc0qA5k0FeyvnfGFKcAsVZ2+b/MPI2/KbB0uY/PWj43H6h9fEqMGVf6 J7Ukas1N2QJBAOIozZjzfs3PQvKHMzOG3mQt+qSjsDKiEq/cPMIRPXYX5pS0+2Gz mtUaikI+Nk5hekXqA2i+4uis9UhCzm+W+d8CQQCEo/ZPrmp1DdnpiNh8hKw+l3Kv jd0lhIyMlrALhfjtqtijhjHEHlo0289DEwv09CtdZFx0koTxqiy7zKiuM/NJAkBZ S2sB/QIQGMmCIMeijJm6TD0uTEMBeuSN8xM6PLxbqEwuYtbuWI/FnFkClrWydOJm QGNgNB47aC7gfSAtBxtZAkBiCJcoVmXkm3rS7scUgCHrNhKH0G0nJbqjgfj57I31 rCw34N6L+fyUozJQCBYdECyI1xG2eajqrMmnVZ046cjF -----END RSA PRIVATE KEY----- memcached-1.5.22/t/chunked-items.t0000664000175000017500000000746613615376577013663 00000000000000#!/usr/bin/perl # Networked logging tests. use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached('-m 48 -o slab_chunk_max=16384'); my $sock = $server->sock; # We're testing to ensure item chaining doesn't corrupt or poorly overlap # data, so create a non-repeating pattern. my @parts = (); for (1 .. 8000) { push(@parts, $_); } my $pattern = join(':', @parts); my $plen = length($pattern); print $sock "set pattern 0 0 $plen\r\n$pattern\r\n"; is(scalar <$sock>, "STORED\r\n", "stored pattern successfully"); mem_get_is($sock, "pattern", $pattern); for (1..5) { my $size = 400 * 1024; my $data = "x" x $size; print $sock "set foo$_ 0 0 $size\r\n$data\r\n"; my $res = <$sock>; is($res, "STORED\r\n", "stored some big items"); } { my $max = 1024 * 1024; my $big = "a big value that's > .5M and < 1M. "; while (length($big) * 2 < $max) { $big = $big . $big; } my $biglen = length($big); for (1..100) { print $sock "set toast$_ 0 0 $biglen\r\n$big\r\n"; is(scalar <$sock>, "STORED\r\n", "stored big"); mem_get_is($sock, "toast$_", $big); } } # Test a wide range of sets. { my $len = 1024 * 200; while ($len < 1024 * 1024) { my $val = "B" x $len; print $sock "set foo_$len 0 0 $len\r\n$val\r\n"; is(scalar <$sock>, "STORED\r\n", "stored size $len"); $len += 2048; } } # Test long appends and prepends. # Note: memory bloats like crazy if we use one test per request. { my $str = 'seedstring'; my $len = length($str); print $sock "set appender 0 0 $len\r\n$str\r\n"; is(scalar <$sock>, "STORED\r\n", "stored seed string for append"); my $unexpected = 0; for my $part (@parts) { # reduce required loops but still have a pattern. my $todo = $part . "x" x 10; $str .= $todo; my $len = length($todo); print $sock "append appender 0 0 $len\r\n$todo\r\n"; is(scalar <$sock>, "STORED\r\n", "append $todo size $len"); print $sock "get appender\r\n"; my $header = scalar <$sock>; my $body = scalar <$sock>; my $end = scalar <$sock>; $unexpected++ unless $body eq "$str\r\n"; } is($unexpected, 0, "No unexpected results during appends\n"); # Now test appending a chunked item to a chunked item. $len = length($str); print $sock "append appender 0 0 $len\r\n$str\r\n"; is(scalar <$sock>, "STORED\r\n", "append large string size $len"); mem_get_is($sock, "appender", $str . $str); print $sock "delete appender\r\n"; is(scalar <$sock>, "DELETED\r\n", "removed appender key"); } { my $str = 'seedstring'; my $len = length($str); print $sock "set prepender 0 0 $len\r\n$str\r\n"; is(scalar <$sock>, "STORED\r\n", "stored seed string for append"); my $unexpected = 0; for my $part (@parts) { # reduce required loops but still have a pattern. $part .= "x" x 10; $str = $part . $str; my $len = length($part); print $sock "prepend prepender 0 0 $len\r\n$part\r\n"; is(scalar <$sock>, "STORED\r\n", "prepend $part size $len"); print $sock "get prepender\r\n"; my $header = scalar <$sock>; my $body = scalar <$sock>; my $end = scalar <$sock>; $unexpected++ unless $body eq "$str\r\n"; } is($unexpected, 0, "No unexpected results during prepends\n"); # Now test prepending a chunked item to a chunked item. $len = length($str); print $sock "prepend prepender 0 0 $len\r\n$str\r\n"; is(scalar <$sock>, "STORED\r\n", "prepend large string size $len"); mem_get_is($sock, "prepender", $str . $str); print $sock "delete prepender\r\n"; is(scalar <$sock>, "DELETED\r\n", "removed prepender key"); } done_testing(); memcached-1.5.22/t/getset.t0000775000175000017500000000617413615376577012414 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 539; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; # set foo (and should get it) print $sock "set foo 0 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); # add bar (and should get it) print $sock "add bar 0 0 6\r\nbarval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored barval"); mem_get_is($sock, "bar", "barval"); # add foo (but shouldn't get new value) print $sock "add foo 0 0 5\r\nfoov2\r\n"; is(scalar <$sock>, "NOT_STORED\r\n", "not stored"); mem_get_is($sock, "foo", "fooval"); # replace bar (should work) print $sock "replace bar 0 0 6\r\nbarva2\r\n"; is(scalar <$sock>, "STORED\r\n", "replaced barval 2"); # replace notexist (shouldn't work) print $sock "replace notexist 0 0 6\r\nbarva2\r\n"; is(scalar <$sock>, "NOT_STORED\r\n", "didn't replace notexist"); # delete foo. print $sock "delete foo\r\n"; is(scalar <$sock>, "DELETED\r\n", "deleted foo"); # delete foo again. not found this time. print $sock "delete foo\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "deleted foo, but not found"); # add moo # print $sock "add moo 0 0 6\r\nmooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored barval"); mem_get_is($sock, "moo", "mooval"); # check-and-set (cas) failure case, try to set value with incorrect cas unique val print $sock "cas moo 0 0 6 0\r\nMOOVAL\r\n"; is(scalar <$sock>, "EXISTS\r\n", "check and set with invalid id"); # test "gets", grab unique ID print $sock "gets moo\r\n"; # VALUE moo 0 6 3084947704 # my @retvals = split(/ /, scalar <$sock>); my $data = scalar <$sock>; # grab data my $dot = scalar <$sock>; # grab dot on line by itself is($retvals[0], "VALUE", "get value using 'gets'"); my $unique_id = $retvals[4]; # clean off \r\n $unique_id =~ s/\r\n$//; ok($unique_id =~ /^\d+$/, "unique ID '$unique_id' is an integer"); # now test that we can store moo with the correct unique id print $sock "cas moo 0 0 6 $unique_id\r\nMOOVAL\r\n"; is(scalar <$sock>, "STORED\r\n"); mem_get_is($sock, "moo", "MOOVAL"); # pipeline is okay print $sock "set foo 0 0 6\r\nfooval\r\ndelete foo\r\nset foo 0 0 6\r\nfooval\r\ndelete foo\r\n"; is(scalar <$sock>, "STORED\r\n", "pipeline set"); is(scalar <$sock>, "DELETED\r\n", "pipeline delete"); is(scalar <$sock>, "STORED\r\n", "pipeline set"); is(scalar <$sock>, "DELETED\r\n", "pipeline delete"); # Test sets up to a large size around 1MB. # Everything up to 1MB - 1k should succeed, everything 1MB +1k should fail. my $len = 1024; while ($len < 1024*1028) { my $val = "B"x$len; if ($len > (1024*1024)) { # Ensure causing a memory overflow doesn't leave stale data. print $sock "set foo_$len 0 0 3\r\nMOO\r\n"; is(scalar <$sock>, "STORED\r\n"); print $sock "set foo_$len 0 0 $len\r\n$val\r\n"; is(scalar <$sock>, "SERVER_ERROR object too large for cache\r\n", "failed to store size $len"); mem_get_is($sock, "foo_$len"); } else { print $sock "set foo_$len 0 0 $len\r\n$val\r\n"; is(scalar <$sock>, "STORED\r\n", "stored size $len"); } $len += 2048; } memcached-1.5.22/t/issue_104.t0000775000175000017500000000114513250303500012571 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 6; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; # first get should miss print $sock "get foo\r\n"; is(scalar <$sock>, "END\r\n", "get foo"); # Now set and get (should hit) print $sock "set foo 0 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); my $stats = mem_stats($sock); is($stats->{cmd_get}, 2, "Should have 2 get requests"); is($stats->{get_hits}, 1, "Should have 1 hit"); is($stats->{get_misses}, 1, "Should have 1 miss"); memcached-1.5.22/t/issue_50.t0000664000175000017500000000060513250303500012506 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 1; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached('-B binary'); my $sock = $server->sock; $SIG{ALRM} = sub { die "alarm\n" }; alarm(2); print $sock "Here's a bunch of garbage that doesn't look like the bin prot."; my $rv = <$sock>; ok(1, "Either the above worked and quit, or hung forever."); memcached-1.5.22/t/error-extstore.t0000664000175000017500000000702213461170555014076 00000000000000#!/usr/bin/perl # Test the "Error on get" path for extstore. # the entire error handling code for process_get_command() never worked, and # would infinite loop. get_extstore() can hit it sometimes. use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $ext_path; if (!supports_extstore()) { plan skip_all => 'extstore not enabled'; exit 0; } $ext_path = "/tmp/extstore.$$"; my $server = new_memcached("-m 64 -I 4m -U 0 -o ext_page_size=8,ext_wbuf_size=8,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,slab_automove=0,ext_compact_under=1"); my $sock = $server->sock; # Wait until all items have flushed sub wait_for_ext { my $sum = 1; while ($sum != 0) { my $s = mem_stats($sock, "items"); $sum = 0; for my $key (keys %$s) { if ($key =~ m/items:(\d+):number/) { # Ignore classes which can contain extstore items next if $1 < 3; $sum += $s->{$key}; } } sleep 1 if $sum != 0; } } # We're testing to ensure item chaining doesn't corrupt or poorly overlap # data, so create a non-repeating pattern. my @parts = (); for (1 .. 8000) { push(@parts, $_); } my $pattern = join(':', @parts); my $plen = length($pattern); # Set some large items and let them flush to extstore. for (1..5) { my $size = 3000 * 1024; my $data = "x" x $size; print $sock "set foo$_ 0 0 $size\r\n$data\r\n"; my $res = <$sock>; is($res, "STORED\r\n", "stored some big items"); } wait_for_ext(); { my $long_key = "f" x 512; print $sock "get foo1 foo2 foo3 $long_key\r\n"; ok(scalar <$sock> =~ /CLIENT_ERROR bad command line format/, 'long key fails'); my $stats = mem_stats($sock); cmp_ok($stats->{get_aborted_extstore}, '>', 1, 'some extstore queries aborted'); } # Disable automatic page balancing, then move enough pages that the large # items can no longer be loaded from extstore { print $sock "slabs automove 0\r\n"; my $res = <$sock>; my $source = 0; while (1) { print $sock "slabs reassign $source 1\r\n"; $res = <$sock>; if ($res =~ m/NOSPARE/) { $source = -1; my $stats = mem_stats($sock, 'slabs'); for my $key (grep { /total_pages/ } keys %$stats) { if ($key =~ m/(\d+):total_pages/) { next if $1 < 3; $source = $1 if $stats->{$key} > 1; } } last if $source == -1; } select undef, undef, undef, 0.10; } } # fetching the large keys should now fail. { print $sock "get foo1\r\n"; my $res = <$sock>; $res =~ s/[\r\n]//g; is($res, 'SERVER_ERROR out of memory writing get response', 'can no longer read back item'); my $stats = mem_stats($sock); is($stats->{get_oom_extstore}, 1, 'check extstore oom counter'); } # Leaving this for future generations. # The process_get_command() function had several memory leaks. my $LEAK_TEST = 0; if ($LEAK_TEST) { my $tries = 0; while ($tries) { print $sock "slabs reassign 1 39\r\n"; my $res = <$sock>; if ($res =~ m/BUSY/) { select undef, undef, undef, 0.10; } else { $tries--; } } my $long_key = "f" x 512; while (1) { print $sock "get foo1 foo2 foo3 $long_key\r\n"; my $res = <$sock>; } } done_testing(); END { unlink $ext_path if $ext_path; } memcached-1.5.22/t/binary-get.t0000755000175000017500000000076312416643766013151 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 8; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $count = 1; foreach my $blob ("mooo\0", "mumble\0\0\0\0\r\rblarg", "\0", "\r") { my $key = "foo$count"; my $len = length($blob); print "len is $len\n"; print $sock "set $key 0 0 $len\r\n$blob\r\n"; is(scalar <$sock>, "STORED\r\n", "stored $key"); mem_get_is($sock, $key, $blob); $count++; } memcached-1.5.22/t/daemonize.t0000755000175000017500000000134212416643766013055 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 7; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use File::Temp qw(tempfile); my (undef, $tmpfn) = tempfile(); my $server = new_memcached("-d -P $tmpfn"); my $sock = $server->sock; sleep 0.5; ok(-e $tmpfn, "pid file exists"); ok(-s $tmpfn, "pid file has length"); open (my $fh, $tmpfn) or die; my $readpid = do { local $/; <$fh>; }; chomp $readpid; close ($fh); ok(kill(0, $readpid), "process is still running"); my $stats = mem_stats($sock); is($stats->{pid}, $readpid, "memcached reports same pid as file"); ok($server->new_sock, "opened new socket"); ok(kill(9, $readpid), "sent KILL signal"); sleep 0.5; ok(! $server->new_sock, "failed to open new socket"); memcached-1.5.22/t/watcher.t0000664000175000017500000000751713611703161012530 00000000000000#!/usr/bin/perl # Networked logging tests. use strict; use warnings; use Socket qw/SO_RCVBUF/; use Test::More tests => 13; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached('-m 60 -o watcher_logbuf_size=8'); my $client = $server->sock; my $watcher = $server->new_sock; # This doesn't return anything. print $watcher "watch\n"; my $res = <$watcher>; is($res, "OK\r\n", "watcher enabled"); print $client "get foo\n"; $res = <$client>; is($res, "END\r\n", "basic get works"); my $spacer = "X"x180; # This is a flaky test... depends on buffer sizes. Could either have memc # shrink the watcher buffer, or loop this and keep doubling until we get some # skipped values. for (1 .. 80000) { print $client "get foo$_$spacer\n"; $res = <$client>; } # Let the logger thread catch up before we start reading. sleep 1; my $do_fetch = 0; #print STDERR "RESULT: $res\n"; while (my $log = <$watcher>) { # The "skipped" line won't actually print until some space frees up in the # buffer, so we need to occasionally cause new lines to generate. if (($do_fetch++ % 100) == 0) { print $client "get foo\n"; $res = <$client>; } next unless $log =~ m/skipped/; like($log, qr/skipped=/, "skipped some lines"); # This should unjam more of the text. print $client "get foob\n"; $res = <$client>; last; } $res = <$watcher>; like($res, qr/ts=\d+\.\d+\ gid=\d+ type=item_get/, "saw a real log line after a skip"); # testing the longest uri encoded key length { my $new_watcher = $server->new_sock; print $new_watcher "watch mutations\n"; my $watch_res = <$new_watcher>; my $key = ""; my $max_keylen = 250; for (1 .. $max_keylen) { $key .= "#"; } print $client "set $key 0 0 9\r\nmemcached\r\n"; $res = <$client>; is ($res, "STORED\r\n", "stored the long key"); if ($res eq "STORED\r\n") { $watch_res = <$new_watcher>; my $max_uri_keylen = $max_keylen * 3 + length("key="); my @tab = split(/\s+/, $watch_res); is (length($tab[3]), $max_uri_keylen, "got the correct uri encoded key length");; } } # test combined logs # fill to evictions, then enable watcher, set again, and look for both lines { my $value = "B"x11000; my $keycount = 8000; for (1 .. $keycount) { print $client "set n,foo$_ 0 0 11000 noreply\r\n$value\r\n"; } $watcher = $server->new_sock; print $watcher "watch mutations evictions\n"; $res = <$watcher>; is($res, "OK\r\n", "new watcher enabled"); my $watcher2 = $server->new_sock; print $watcher2 "watch evictions\n"; $res = <$watcher2>; is($res, "OK\r\n", "evictions watcher enabled"); print $client "set bfoo 0 0 11000 noreply\r\n$value\r\n"; my $found_log = 0; my $found_ev = 0; while (my $log = <$watcher>) { $found_log = 1 if ($log =~ m/type=item_store/); $found_ev = 1 if ($log =~ m/type=eviction/); last if ($found_log && $found_ev); } is($found_log, 1, "found rawcmd log entry"); is($found_ev, 1, "found eviction log entry"); } # test cas command logs { $watcher = $server->new_sock; print $watcher "watch mutations\n"; $res = <$watcher>; is($res, "OK\r\n", "mutations watcher enabled"); print $client "cas cas_watch_key 0 0 5 0\r\nvalue\r\n"; my $tries = 30; my $found_cas = 0; while (my $log = <$watcher>) { $found_cas = 1 if ($log =~ m/cmd=cas/ && $log =~ m/cas_watch_key/); last if ($tries-- == 0 || $found_cas); sleep 1; } is($found_cas, 1, "correctly logged cas command"); } # test no_watch option { my $nowatch_server = new_memcached('-W'); my $watchsock = $nowatch_server->new_sock; print $watchsock "watch mutations\n"; my $watchresult = <$watchsock>; is($watchresult, "CLIENT_ERROR watch commands not allowed\r\n", "attempted watch gave client error with no_watch option set"); } memcached-1.5.22/t/bogus-commands.t0000755000175000017500000000041712416643766014022 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 1; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; print $sock "boguscommand slkdsldkfjsd\r\n"; is(scalar <$sock>, "ERROR\r\n", "got error back"); memcached-1.5.22/t/slabs-reassign-chunked.t0000664000175000017500000000766313461170555015441 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More tests => 8; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached('-m 60 -o slab_reassign,slab_automove,lru_crawler,lru_maintainer,slab_chunk_max=4096'); my $sock = $server->sock; sub dump_stats { my $s = shift; my $filter = shift || ''; for my $k (sort keys %$s) { if ($filter) { next unless $k =~ m/$filter/; } print STDERR "STAT: $k = ", $s->{$k}, "\n"; } } my $value; { my @chars = ("C".."Z"); for (1 .. 11000) { $value .= $chars[rand @chars]; } } my $keycount = 5100; my $res; for (1 .. $keycount) { # print STDERR "HI $_\n"; print $sock "set nfoo$_ 0 0 11000 noreply\r\n$value\r\n"; # print $sock "set nfoo$_ 0 0 11000\r\n$value\r\n"; # my $res = scalar <$sock>; # print STDERR "RES: $res\n"; } my $todelete = 0; { my $stats = mem_stats($sock); cmp_ok($stats->{curr_items}, '>', 4000, "stored at least 4000 11k items"); $todelete = $stats->{curr_items} / 2; # for ('evictions', 'reclaimed', 'curr_items', 'cmd_set', 'bytes') { # print STDERR "$_: ", $stats->{$_}, "\n"; # } } { my $s = mem_stats($sock, 'slabs'); my $sid; # Find the highest ID to source from. for my $k (keys %$s) { next unless $k =~ m/^(\d+):/; $sid = $s->{$k} if $s->{$k} > $1; } for (my $x = 0; $x < 3; $x++) { print $sock "slabs reassign 17 0\r\n"; my $res = scalar <$sock>; chomp $res; # print STDERR "SLABS REASSIGN RESULT: $res\n"; sleep 1; } } # Make room in old class so rescues can happen when we switch slab classes. #for (1 .. $todelete) { # print $sock "delete nfoo$_ noreply\r\n"; #} # Give LRU mover some time to reclaim slab chunks. #sleep 1; { my $stats = mem_stats($sock); cmp_ok($stats->{slab_global_page_pool}, '>', 0, 'global page pool > 0'); cmp_ok($stats->{slab_reassign_chunk_rescues}, '>', 0, 'some chunk rescues happened'); } { my $hits = 0; for (1 .. $keycount) { print $sock "get nfoo$_\r\n"; my $body = scalar(<$sock>); my $expected = "VALUE nfoo$_ 0 11000\r\n$value\r\nEND\r\n"; if ($body =~ /^END/) { next; } else { $body .= scalar(<$sock>) . scalar(<$sock>); if ($body ne $expected) { die "Something terrible has happened: $expected\nBODY:\n$body\nDONETEST\n"; } $hits++; } } cmp_ok($hits, '>', 0, "fetched back $hits values after reassignment"); } $value = "A"x3000; for (1 .. $keycount) { print $sock "set ifoo$_ 0 0 3000 noreply\r\n$value\r\n"; } my $missing = 0; my $hits = 0; for (1 .. $keycount) { print $sock "get ifoo$_\r\n"; my $body = scalar(<$sock>); my $expected = "VALUE ifoo$_ 0 3000\r\n$value\r\nEND\r\n"; if ($body =~ /^END/) { $missing++; } else { $body .= scalar(<$sock>) . scalar(<$sock>); if ($body ne $expected) { print STDERR "Something terrible has happened: $expected\nBODY:\n$body\nDONETEST\n"; } else { $hits++; } } } #print STDERR "HITS: $hits, MISSES: $missing\n"; { my $stats = mem_stats($sock); cmp_ok($stats->{evictions}, '<', 2000, 'evictions were less than 2000'); # for ('evictions', 'reclaimed', 'curr_items', 'cmd_set', 'bytes') { # print STDERR "$_: ", $stats->{$_}, "\n"; # } } cmp_ok($hits, '>', 4000, 'were able to fetch back 2/3rds of 8k keys'); my $stats_done = mem_stats($sock); cmp_ok($stats_done->{slab_reassign_chunk_rescues}, '>', 0, 'some reassign chunk rescues happened'); # Reassign rescues won't happen here because the headers are of a different # size and we aren't moving pages out of that slab class #cmp_ok($stats_done->{slab_reassign_rescues}, '>', 0, 'some reassign rescues happened'); cmp_ok($stats_done->{slab_reassign_evictions_nomem}, '>', 0, 'some reassign evictions happened'); memcached-1.5.22/t/issue_260.t0000775000175000017500000000450113461170555012613 00000000000000#!/usr/bin/perl # Issue #260 is a terrible bug. # In order to run this test: # * checkout 1.4.15 # * change TAIL_REPAIR_TIME from (3 * 3600) to 3 # Now it should cause an assert. Patches can be tested to fix it. use strict; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; plan skip_all => "Only possible to test #260 under artificial conditions"; exit 0; plan tests => 11074; # assuming max slab is 1M and default mem is 64M my $server = new_memcached(); my $sock = $server->sock; # create a big value for the largest slab my $max = 1024 * 1024; my $big = 'x' x (1024 * 1024 - 250); ok(length($big) > 512 * 1024); ok(length($big) < 1024 * 1024); # set the big value my $len = length($big); print $sock "set big 0 0 $len\r\n$big\r\n"; is(scalar <$sock>, "STORED\r\n", "stored big"); mem_get_is($sock, "big", $big); # no evictions yet my $stats = mem_stats($sock); is($stats->{"evictions"}, "0", "no evictions to start"); # set many big items, enough to get evictions for (my $i = 0; $i < 100; $i++) { print $sock "set item_$i 0 0 $len\r\n$big\r\n"; is(scalar <$sock>, "STORED\r\n", "stored item_$i"); } # some evictions should have happened my $stats = mem_stats($sock); my $evictions = int($stats->{"evictions"}); ok($evictions == 37, "some evictions happened"); # the first big value should be gone mem_get_is($sock, "big", undef); # the earliest items should be gone too for (my $i = 0; $i < $evictions - 1; $i++) { mem_get_is($sock, "item_$i", undef); } # check that the non-evicted are the right ones for (my $i = $evictions - 1; $i < $evictions + 4; $i++) { mem_get_is($sock, "item_$i", $big); } # Now we fill a slab with incrementable items... for (my $i = 0; $i < 10923; $i++) { print $sock "set sitem_$i 0 0 1\r\n1\r\n"; is(scalar <$sock>, "STORED\r\n", "stored sitem_$i"); } my $stats = mem_stats($sock); my $evictions = int($stats->{"evictions"}); ok($evictions == 38, "one more eviction happened: $evictions"); # That evicted item was the first one we put in. mem_get_is($sock, "sitem_0", undef); sleep 15; # Now we increment the item which should be on the tail. # THIS asserts the memcached-debug binary. print $sock "incr sitem_1 1\r\n"; is(scalar <$sock>, "2\r\n", "incremented to two"); #my $stats = mem_stats($sock, "slabs"); #is($stats->{"1:free_chunks"}, 0, "free chunks should still be 0"); memcached-1.5.22/t/restart.t0000664000175000017500000000731713575022500012555 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; # NOTE: Do not use this feature on top of a filesystem, please use a ram disk! # These tests use /tmp/ as some systems do not have or have a weirdly small # /dev/shm. my $mem_path = "/tmp/mc_restart.$$"; # read a invalid metadata file { my $meta_path = "$mem_path.meta"; open(my $f, "> $meta_path") || die("Can't open a metadata file."); eval { new_memcached("-e $mem_path"); }; unlink($meta_path); ok($@, "Died with an empty metadata file"); } my $server = new_memcached("-m 128 -e $mem_path -I 2m"); my $sock = $server->sock; diag "Set some values, various sizes."; { my $cur = 2; my $cnt = 0; my $end = 2**20; while ($cur <= $end) { my $val = 'x' x $cur; print $sock "set foo${cnt} 0 0 $cur\r\n$val\r\n"; like(scalar <$sock>, qr/STORED/, "stored $cur size item"); $cur *= 2; $cnt++; } } diag "load enough items to change hash power level"; { my $stats = mem_stats($sock); is($stats->{hash_power_level}, 16, "starting hash level is 16"); my $todo = 2**17; my $good = 1; while ($todo--) { print $sock "set z${todo} 0 0 0\r\n\r\n"; my $res = <$sock>; $good = 0 if ($res !~ m/STORED/); } is($good, 1, "set responses were all STORED"); sleep 3; # sigh. $stats = mem_stats($sock); is($stats->{hash_power_level}, 17, "new hash power level is 17"); # Now delete all these items, so the auto-restore won't cause the hash # table to re-inflate, but the restart code should restore the hash table # to where it was regardless. $todo = 2**17; $good = 1; while ($todo--) { print $sock "delete z${todo}\r\n"; my $res = <$sock>; $good = 0 if ($res !~ m/DELETED/); } is($good, 1, "delete responses were all DELETED"); } diag "Load a couple chunked items"; { my $cur = 768000; my $cnt = 0; my $end = $cur + 1024; while ($cur <= $end) { my $val = 'x' x $cur; print $sock "set chunk${cnt} 0 0 $cur\r\n$val\r\n"; like(scalar <$sock>, qr/STORED/, "stored $cur size item"); $cur += 50; $cnt++; } } diag "Data that should expire while stopped."; { print $sock "set low1 0 8 2\r\nbo\r\n"; like(scalar <$sock>, qr/STORED/, "stored low ttl item"); # This one should stay. print $sock "set low2 0 20 2\r\nmo\r\n"; like(scalar <$sock>, qr/STORED/, "stored low ttl item"); } # make sure it's okay to stop with a logger watcher enabled. { my $wsock = $server->new_sock; print $wsock "watch fetchers mutations\n"; my $res = <$wsock>; is($res, "OK\r\n", "watcher enabled"); } $server->graceful_stop(); diag "killed, waiting"; # TODO: add way to wait for server to fully exit.. sleep 10; { $server = new_memcached("-m 128 -e $mem_path -I 2m"); $sock = $server->sock; diag "reconnected"; my $stats = mem_stats($sock); is($stats->{hash_power_level}, 17, "restarted hash power level is 17"); } diag "low TTL item should be gone"; { mem_get_is($sock, 'low1', undef); # but this one should exist. mem_get_is($sock, 'low2', 'mo'); } # initially inserted items. { my $cur = 2; my $cnt = 0; my $end = 1000000; while ($cur < $end) { my $val = 'x' x $cur; mem_get_is($sock, 'foo' . $cnt, $val); $cur *= 2; $cnt++; } } # chunked items. { my $cur = 768000; my $cnt = 0; my $end = $cur + 1024; while ($cur <= $end) { my $val = 'x' x $cur; mem_get_is($sock, 'chunk' . $cnt, $val); $cur += 50; $cnt++; } } done_testing(); END { unlink $mem_path if $mem_path; } memcached-1.5.22/t/server.pem0000664000175000017500000001156513575022500012715 00000000000000subject= C = UK, O = OpenSSL Group, OU = FOR TESTING PURPOSES ONLY, CN = Test Server Cert issuer= C = UK, O = OpenSSL Group, OU = FOR TESTING PURPOSES ONLY, CN = OpenSSL Test Intermediate CA -----BEGIN CERTIFICATE----- MIID0DCCArigAwIBAgIIcsOElVeHzfYwDQYJKoZIhvcNAQELBQAwcDELMAkGA1UE BhMCVUsxFjAUBgNVBAoMDU9wZW5TU0wgR3JvdXAxIjAgBgNVBAsMGUZPUiBURVNU SU5HIFBVUlBPU0VTIE9OTFkxJTAjBgNVBAMMHE9wZW5TU0wgVGVzdCBJbnRlcm1l ZGlhdGUgQ0EwIBcNMTgwNjE0MTI0NjI4WhgPMjExODA2MTQxMjQ2MjhaMGQxCzAJ BgNVBAYTAlVLMRYwFAYDVQQKDA1PcGVuU1NMIEdyb3VwMSIwIAYDVQQLDBlGT1Ig VEVTVElORyBQVVJQT1NFUyBPTkxZMRkwFwYDVQQDDBBUZXN0IFNlcnZlciBDZXJ0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0jIZ8IZ4dIzBc+ZfdmG5 n8G3JzRX99QvIqv52s4hFVfdzoa+AciKJpo9zkegWPmfsAVNa4uVceg/ZQt6qJsu G/pxbQSZVnyjDQGtt7rgaDEbyUP0XJCnzyRdWSUjFS8yNZn4NkmZU01GlHtXdzWy dEa5PaiTIwW0HI+bjjOEhwJ1hFuFqzlKHVKHA6DBzNcl6ly0E/q2kyslbR+0hq7p NMqKvvuAxqgc//W8KvLDlKAt9D3t5zgh2+BrMPemrzjEaM97yHTogJo7+SKVDdUw YQ7Br3xfyki9u2bUYib1BMSvLezxNP0qf/iU91z4xyLmMvOXE6W0D1WHwya1CfE7 vwIDAQABo3gwdjAdBgNVHQ4EFgQU3ulCbvgfxej6rHnddMpBidwnLIIwHwYDVR0j BBgwFoAUCgNEpWg658NdblSLsvg43EA1WwUwCQYDVR0TBAIwADATBgNVHSUEDDAK BggrBgEFBQcDATAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQAD ggEBAENMzaqJtmWED++W4KXFVwNBkQ87errBXe4jVeYKpjNb0JGMm60MS5ty54fb r27SsR2EEk3EK2rcd85RR7TEKZCn9SvPykVtVf0tru7nOptQJgSbRvxIzyyq1UcE K+BXDgN/I0f1X6qbk4Stb6uJF7yyAUabacjwKqgVifOOeKF9WJhVA8qJKoVq7HLN k+uvm0geO1I4LKeULXVnQy8kwB6twcxN8iPyO45ZxbYIVeEKaYtbj/XPoq6KsLIb 5fj+mK1r/LkWk352ksNhf73r3alF8TBcSLqnbMoy1/ZvzlI4ksp9IGWtIU+CzP/f VUjh00NOwDLd5jJbPoWW0oNp9m4= -----END CERTIFICATE----- subject= C = UK, O = OpenSSL Group, OU = FOR TESTING PURPOSES ONLY, CN = OpenSSL Test Intermediate CA issuer= C = UK, O = OpenSSL Group, OU = FOR TESTING PURPOSES ONLY, CN = OpenSSL Test Root CA -----BEGIN CERTIFICATE----- MIIEPzCCAqegAwIBAgIILsaQqJAjK4IwDQYJKoZIhvcNAQELBQAwaDELMAkGA1UE BhMCVUsxFjAUBgNVBAoMDU9wZW5TU0wgR3JvdXAxIjAgBgNVBAsMGUZPUiBURVNU SU5HIFBVUlBPU0VTIE9OTFkxHTAbBgNVBAMMFE9wZW5TU0wgVGVzdCBSb290IENB MCAXDTE4MDYxNDEyNDYyOFoYDzIxMTgwNjE0MTI0NjI4WjBwMQswCQYDVQQGEwJV SzEWMBQGA1UECgwNT3BlblNTTCBHcm91cDEiMCAGA1UECwwZRk9SIFRFU1RJTkcg UFVSUE9TRVMgT05MWTElMCMGA1UEAwwcT3BlblNTTCBUZXN0IEludGVybWVkaWF0 ZSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANIpVng2wNFJp2kF oJ6Yji25wy1YufnS8NxA82fk5OHdhGWj1CWqnQNotEqEQzcOUszQYrNxd8tEvoWk Ik4JMBVoEcgBGedchftptTNulFWodWpi1yFaqA/Nz2BsVgcCJW4C+UWDT7VeHtGU 7tYKKr35lxp6io/a4jUDQXvO2nJA9YlrxOktunMqtoZSYqUz35ZXsdkn58o8Fbqm dEpw6AqAr9aBgY5DSaGxbaX2lwNt9NvB+f9ucOqEnPP8AfTlPYc/ENwJ6u/H8RGw d1im71mu2lHjcws3aHkbluH860U3vlKWx6Ff1qdQcH98e2HwElqxCK00xya8leu4 u64nljkCAwEAAaNjMGEwHQYDVR0OBBYEFAoDRKVoOufDXW5Ui7L4ONxANVsFMB8G A1UdIwQYMBaAFDZjTeLsQUG6KL9xuLhzXVdB4pkKMA8GA1UdEwEB/wQFMAMBAf8w DgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3DQEBCwUAA4IBgQDZQJLA90ewVaS3E3du gSjPkQ1xsHm8H1am+7zr5oZ81J+R8XYIZgMR+9ShVo38OradiYNqDLso+4iuVdxh hzoSoQELoDXCficzWKnlAtWvwDDoczyK+/p94g3VKx14n2+GvQzoZ4kwQQgaFH1w YI6w0oH9zwoklCxvihj8D069QrYyuTT8JGZ2m0FHqVJg6teuQKFahSgwYR2CUoIb 6PrpSUQeCVCH8TPkzlRT6UgtM3ERt7+TlQ+zZ80dSf4YTAsDv9Z/CJXiF/5wZr6/ lWuFjWmX2HkpEW6Wiv5KF8QP6Ft7Z+RYua7RMtELCYvqYbWDBs7fXWGBkZ5xhB09 jCxz+F7zOeRbyzacfFq9DhxCWCRbIrdgGGE/Of2ujJtmK/2p4M6E5IsKNAI2SJBW iJXvIgQgR22ehPqy6er2Gog5LkWUwqB0kHZJJpbp1IW01IGTpD6YAJyVCEAlyMbo Kto9+wQFLT3Auv/W5h6OwxkNdfAyZBYy0ZSFk4EE8OdWWY4= -----END CERTIFICATE----- -----BEGIN RSA PRIVATE KEY----- MIIEpQIBAAKCAQEA0jIZ8IZ4dIzBc+ZfdmG5n8G3JzRX99QvIqv52s4hFVfdzoa+ AciKJpo9zkegWPmfsAVNa4uVceg/ZQt6qJsuG/pxbQSZVnyjDQGtt7rgaDEbyUP0 XJCnzyRdWSUjFS8yNZn4NkmZU01GlHtXdzWydEa5PaiTIwW0HI+bjjOEhwJ1hFuF qzlKHVKHA6DBzNcl6ly0E/q2kyslbR+0hq7pNMqKvvuAxqgc//W8KvLDlKAt9D3t 5zgh2+BrMPemrzjEaM97yHTogJo7+SKVDdUwYQ7Br3xfyki9u2bUYib1BMSvLezx NP0qf/iU91z4xyLmMvOXE6W0D1WHwya1CfE7vwIDAQABAoIBAQC2HAo1RYvfDoQc sh9LJWf5bZANO2Brqz4bP/x9AdHP+AyH/l1oliJ7R2785TmbXMppam6lGo4j3h/u n39pzOip/NWAqldfgySRBD9Jy3LZUpLMUT/JYtrAsLTfozk+BWHu5rMR9boNXgok Yqho8/DkpNGhBghUc4CUricLkL7laD3ziAHpx8yALL3tnLGOpgT9hNrA8Dm3yfUS JEfiG12ILXvq1IP+vUNuaLpTLJZuqUmLpK8v+CBYgKxfd+TDnEjul4PqhhIIFK3A xEZYQR2D/AXUwng9hP9uCbVm5lOY6vRbi9Fpbt+KRv+m25s1AnuhJFBOsL30h/Tb iCKWm/nhAoGBAO0bFqMvZHjaT2KiwOwG/Ze9NsjynFPVltiuCqNj8HE5wM6imC5J SdB+jMkgN6ERXALWrtr8Uf2pqzfeMsi6pekOOVTWLe/8c4bAZRxaCZn/BlZRysZI vB9Gb7m7Oymw5iDSqrYywgOiUu+oIiCrmPOealhmn7zmHzHaETvdL9zDAoGBAOLy DVT1csoexnuHVIWqnp7FK7lv6eOGZSdXpfJ3XYjmKJLK2hpVZe+J/mFOL1wsKSt4 0k/V0dnkHR7V4Pa4ECiCthkWMWrBVIHe7+ZnZ0ocKQSC+EEecavOiZ57S/qnUlT6 NtQP4cSy4DHzzFZdTZnn+2oymapPZpb2mvSN/GVVAoGADrIlHwwq8Aqn7Pclefuc 8DC8GoxfABs29EslQadKGdp4htYxFH1aY9/UHgsvJ36J82sW/1+wPUas5BOTljlr WxyUlRuJUVyWVH3MRouWGMNjwynipZOQhWe6OQrPye+688Ha7twKhmsjNNN4+glo u4DQGpaRxAWHXXGkq88zzj0CgYEAsICEceD7R8srnwMfb13FQ8IhQXWSuAvcO/7k 53CCZGhsgc4WVoi4YNY360G9f7gwxMiQ+NpY/Vd2dnbtIbUBjCAss9IY2OhHa0IR 3mXpZTAFjqa1oR+mVHKrgYBvFSBw3fpEDiXT9wEPcIomD709D0fmty9nZ5edOCfP WAfdlokCgYEAqXuMuAg3NMMgEv+eBfsf43v3hRwBqPYanE26wcO3GoT/S8BpB6wy vBoPZOlO5ZfsD2jaTec60GLay+MofxC7qNXIjzHOw50ry4bqHqqoQbn2cONE1k+0 ov7H2keTcG9FEGgL7dRUq3pRUo/W12WmRuDN17IEgkzAeisJnoiPtaQ= -----END RSA PRIVATE KEY----- memcached-1.5.22/t/issue_14.t0000664000175000017500000000144613250303500012512 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 21; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $value = "B"x66560; my $key = 0; for ($key = 0; $key < 10; $key++) { print $sock "set key$key 0 2 66560\r\n$value\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key$key"); } #print $sock "stats slabs" my $first_stats = mem_stats($sock, "slabs"); my $first_malloc = $first_stats->{total_malloced}; sleep(4); for ($key = 10; $key < 20; $key++) { print $sock "set key$key 0 2 66560\r\n$value\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key$key"); } my $second_stats = mem_stats($sock, "slabs"); my $second_malloc = $second_stats->{total_malloced}; is ($second_malloc, $first_malloc, "Memory grows..") memcached-1.5.22/t/binary.t0000775000175000017500000006130513575022500012355 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached("-o no_modern"); ok($server, "started the server"); # Based almost 100% off testClient.py which is: # Copyright (c) 2007 Dustin Sallings # Command constants use constant CMD_GET => 0x00; use constant CMD_SET => 0x01; use constant CMD_ADD => 0x02; use constant CMD_REPLACE => 0x03; use constant CMD_DELETE => 0x04; use constant CMD_INCR => 0x05; use constant CMD_DECR => 0x06; use constant CMD_QUIT => 0x07; use constant CMD_FLUSH => 0x08; use constant CMD_GETQ => 0x09; use constant CMD_NOOP => 0x0A; use constant CMD_VERSION => 0x0B; use constant CMD_GETK => 0x0C; use constant CMD_GETKQ => 0x0D; use constant CMD_APPEND => 0x0E; use constant CMD_PREPEND => 0x0F; use constant CMD_STAT => 0x10; use constant CMD_SETQ => 0x11; use constant CMD_ADDQ => 0x12; use constant CMD_REPLACEQ => 0x13; use constant CMD_DELETEQ => 0x14; use constant CMD_INCREMENTQ => 0x15; use constant CMD_DECREMENTQ => 0x16; use constant CMD_QUITQ => 0x17; use constant CMD_FLUSHQ => 0x18; use constant CMD_APPENDQ => 0x19; use constant CMD_PREPENDQ => 0x1A; use constant CMD_TOUCH => 0x1C; use constant CMD_GAT => 0x1D; use constant CMD_GATQ => 0x1E; use constant CMD_GATK => 0x23; use constant CMD_GATKQ => 0x24; # REQ and RES formats are divided even though they currently share # the same format, since they _could_ differ in the future. use constant REQ_PKT_FMT => "CCnCCnNNNN"; use constant RES_PKT_FMT => "CCnCCnNNNN"; use constant INCRDECR_PKT_FMT => "NNNNN"; use constant MIN_RECV_BYTES => length(pack(RES_PKT_FMT)); use constant REQ_MAGIC => 0x80; use constant RES_MAGIC => 0x81; my $mc = MC::Client->new; # Let's turn on detail stats for all this stuff $mc->stats('detail on'); my $check = sub { my ($key, $orig_flags, $orig_val) = @_; my ($flags, $val, $cas) = $mc->get($key); is($flags, $orig_flags, "Flags is set properly"); ok($val eq $orig_val || $val == $orig_val, $val . " = " . $orig_val); }; my $set = sub { my ($key, $exp, $orig_flags, $orig_value) = @_; $mc->set($key, $orig_value, $orig_flags, $exp); $check->($key, $orig_flags, $orig_value); }; my $empty = sub { my $key = shift; my $rv =()= eval { $mc->get($key) }; is($rv, 0, "Didn't get a result from get"); ok($@->not_found, "We got a not found error when we expected one"); }; my $delete = sub { my ($key, $when) = @_; $mc->delete($key, $when); $empty->($key); }; # diag "Test Version"; my $v = $mc->version; ok(defined $v && length($v), "Proper version: $v"); # Bug 71 { my %stats1 = $mc->stats(''); $mc->flush; my %stats2 = $mc->stats(''); is($stats2{'cmd_flush'}, $stats1{'cmd_flush'} + 1, "Stats not updated on a binary flush"); } # diag "Flushing..."; $mc->flush; # diag "Noop"; $mc->noop; # diag "Simple set/get"; $set->('x', 5, 19, "somevalue"); # diag "Delete"; $delete->('x'); # diag "Flush"; $set->('x', 5, 19, "somevaluex"); $set->('y', 5, 17, "somevaluey"); $mc->flush; $empty->('x'); $empty->('y'); { diag "Some chunked item tests"; my $s2 = new_memcached('-o no_modern,slab_chunk_max=4096'); ok($s2, "started the server"); my $m2 = MC::Client->new($s2); # Specifically trying to cross the chunk boundary when internally # appending CLRF. for my $k (7900..8100) { my $val = 'd' x $k; $val .= '123'; $m2->set('t', $val, 0, 0); # Ensure we get back the same value. Bugs can chop chars. my (undef, $gval, undef) = $m2->get('t'); ok($gval eq $val, $gval . " = " . $val); } my $cval = ('d' x 8100) . '123'; my $m3 = $s2->new_sock; mem_get_is($m3, 't', $cval, "large value set from bin fetched from ascii"); } { # diag "Add"; $empty->('i'); $mc->add('i', 'ex', 5, 10); $check->('i', 5, "ex"); my $rv =()= eval { $mc->add('i', "ex2", 10, 5) }; is($rv, 0, "Add didn't return anything"); ok($@->exists, "Expected exists error received"); $check->('i', 5, "ex"); } { # diag "Too big."; $empty->('toobig'); $mc->set('toobig', 'not too big', 10, 10); eval { my $bigval = ("x" x (1024*1024)) . "x"; $mc->set('toobig', $bigval, 10, 10); }; ok($@->too_big, "Was too big"); $empty->('toobig'); } { # diag "Replace"; $empty->('j'); my $rv =()= eval { $mc->replace('j', "ex", 19, 5) }; is($rv, 0, "Replace didn't return anything"); ok($@->not_found, "Expected not_found error received"); $empty->('j'); $mc->add('j', "ex2", 14, 5); $check->('j', 14, "ex2"); $mc->replace('j', "ex3", 24, 5); $check->('j', 24, "ex3"); } { # diag "MultiGet"; $mc->add('xx', "ex", 1, 5); $mc->add('wye', "why", 2, 5); my $rv = $mc->get_multi(qw(xx wye zed)); # CAS is returned with all gets. $rv->{xx}->[2] = 0; $rv->{wye}->[2] = 0; is_deeply($rv->{xx}, [1, 'ex', 0], "X is correct"); is_deeply($rv->{wye}, [2, 'why', 0], "Y is correct"); is(keys(%$rv), 2, "Got only two answers like we expect"); } # diag "Test increment"; $mc->flush; is($mc->incr("x"), 0, "First incr call is zero"); is($mc->incr("x"), 1, "Second incr call is one"); is($mc->incr("x", 211), 212, "Adding 211 gives you 212"); is($mc->incr("x", 2**33), 8589934804, "Blast the 32bit border"); # diag "Issue 48 - incrementing plain text."; { $mc->set("issue48", "text", 0, 0); my $rv =()= eval { $mc->incr('issue48'); }; ok($@ && $@->delta_badval, "Expected invalid value when incrementing text."); $check->('issue48', 0, "text"); $rv =()= eval { $mc->decr('issue48'); }; ok($@ && $@->delta_badval, "Expected invalid value when decrementing text."); $check->('issue48', 0, "text"); } # diag "Issue 320 - incr/decr wrong length for initial value"; { $mc->flush; is($mc->incr("issue320", 1, 1, 0), 1, "incr initial value is 1"); my (undef, $rv, undef) = $mc->get("issue320"); is(length($rv), 1, "initial value length is 1"); is($rv, "1", "initial value is 1"); } # diag "Test decrement"; $mc->flush; is($mc->incr("x", undef, 5), 5, "Initial value"); is($mc->decr("x"), 4, "Decrease by one"); is($mc->decr("x", 211), 0, "Floor is zero"); { # diag "bug220 my ($rv, $cas) = $mc->set("bug220", "100", 0, 0); my ($irv, $icas) = $mc->incr_cas("bug220", 999); ok($icas != $cas); is($irv, 1099, "Incr amount failed"); my ($flags, $val, $gcas) = $mc->get("bug220"); is($gcas, $icas, "CAS didn't match after incr/gets"); ($irv, $icas) = $mc->incr_cas("bug220", 999); ok($icas != $cas); is($irv, 2098, "Incr amount failed"); ($flags, $val, $gcas) = $mc->get("bug220"); is($gcas, $icas, "CAS didn't match after incr/gets"); } { # diag "bug21"; $mc->add("bug21", "9223372036854775807", 0, 0); is($mc->incr("bug21"), 9223372036854775808, "First incr for bug21."); is($mc->incr("bug21"), 9223372036854775809, "Second incr for bug21."); is($mc->decr("bug21"), 9223372036854775808, "Decr for bug21."); } { # diag "CAS"; $mc->flush; { my $rv =()= eval { $mc->set("x", "bad value", 19, 5, 0x7FFFFFF) }; is($rv, 0, "Empty return on expected failure"); ok($@->not_found, "Error was 'not found' as expected"); } my ($r, $rcas) = $mc->add("x", "original value", 5, 19); my ($flags, $val, $i) = $mc->get("x"); is($val, "original value", "->gets returned proper value"); is($rcas, $i, "Add CAS matched."); { my $rv =()= eval { $mc->set("x", "broken value", 19, 5, $i+1) }; is($rv, 0, "Empty return on expected failure (1)"); ok($@->exists, "Expected error state of 'exists' (1)"); } ($r, $rcas) = $mc->set("x", "new value", 19, 5, $i); my ($newflags, $newval, $newi) = $mc->get("x"); is($newval, "new value", "CAS properly overwrote value"); is($rcas, $newi, "Get CAS matched."); { my $rv =()= eval { $mc->set("x", "replay value", 19, 5, $i) }; is($rv, 0, "Empty return on expected failure (2)"); ok($@->exists, "Expected error state of 'exists' (2)"); } } # diag "Touch commands"; { $mc->flush; $mc->set("totouch", "toast", 0, 1); my $res = $mc->touch("totouch", 10); sleep 2; $check->("totouch", 0, "toast"); $mc->set("totouch", "toast2", 0, 1); my ($flags, $val, $i) = $mc->gat("totouch", 10); is($val, "toast2", "GAT returned correct value"); sleep 2; $check->("totouch", 0, "toast2"); # Test miss as well $mc->set("totouch", "toast3", 0, 1); $res = $mc->touch("totouch", 1); sleep 3; $empty->("totouch"); } # diag "Silent set."; $mc->silent_mutation(::CMD_SETQ, 'silentset', 'silentsetval'); # diag "Silent add."; $mc->silent_mutation(::CMD_ADDQ, 'silentadd', 'silentaddval'); # diag "Silent replace."; { my $key = "silentreplace"; my $extra = pack "NN", 829, 0; $empty->($key); # $mc->send_silent(::CMD_REPLACEQ, $key, 'somevalue', 7278552, $extra, 0); # $empty->($key); $mc->add($key, "xval", 831, 0); $check->($key, 831, 'xval'); $mc->send_silent(::CMD_REPLACEQ, $key, 'somevalue', 7278552, $extra, 0); $check->($key, 829, 'somevalue'); } # diag "Silent delete"; { my $key = "silentdelete"; $empty->($key); $mc->set($key, "some val", 19, 0); $mc->send_silent(::CMD_DELETEQ, $key, '', 772); $empty->($key); } # diag "Silent increment"; { my $key = "silentincr"; my $opaque = 98428747; $empty->($key); $mc->silent_incrdecr(::CMD_INCREMENTQ, $key, 0, 0, 0); is($mc->incr($key, 0), 0, "First call is 0"); $mc->silent_incrdecr(::CMD_INCREMENTQ, $key, 8, 0, 0); is($mc->incr($key, 0), 8); } # diag "Silent decrement"; { my $key = "silentdecr"; my $opaque = 98428147; $empty->($key); $mc->silent_incrdecr(::CMD_DECREMENTQ, $key, 0, 185, 0); is($mc->incr($key, 0), 185); $mc->silent_incrdecr(::CMD_DECREMENTQ, $key, 8, 0, 0); is($mc->incr($key, 0), 177); } # diag "Silent flush"; { my %stats1 = $mc->stats(''); $set->('x', 5, 19, "somevaluex"); $set->('y', 5, 17, "somevaluey"); $mc->send_silent(::CMD_FLUSHQ, '', '', 2775256); $empty->('x'); $empty->('y'); my %stats2 = $mc->stats(''); is($stats2{'cmd_flush'}, $stats1{'cmd_flush'} + 1, "Stats not updated on a binary quiet flush"); } # diag "Append"; { my $key = "appendkey"; my $value = "some value"; $set->($key, 8, 19, $value); $mc->_append_prepend(::CMD_APPEND, $key, " more"); $check->($key, 19, $value . " more"); } # diag "Prepend"; { my $key = "prependkey"; my $value = "some value"; $set->($key, 8, 19, $value); $mc->_append_prepend(::CMD_PREPEND, $key, "prefixed "); $check->($key, 19, "prefixed " . $value); } # diag "Silent append"; { my $key = "appendqkey"; my $value = "some value"; $set->($key, 8, 19, $value); $mc->send_silent(::CMD_APPENDQ, $key, " more", 7284492); $check->($key, 19, $value . " more"); } # diag "Silent prepend"; { my $key = "prependqkey"; my $value = "some value"; $set->($key, 8, 19, $value); $mc->send_silent(::CMD_PREPENDQ, $key, "prefixed ", 7284492); $check->($key, 19, "prefixed " . $value); } # diag "Leaky binary get test."; # # http://code.google.com/p/memcached/issues/detail?id=16 { # Get a new socket so we can speak text to it. my $sock = $server->new_sock; my $max = 1024 * 1024; my $big = "a big value that's > .5M and < 1M. "; while (length($big) * 2 < $max) { $big = $big . $big; } my $biglen = length($big); for(1..100) { my $key = "some_key_$_"; # print STDERR "Key is $key\n"; # print $sock "set $key 0 0 $vallen\r\n$value\r\n"; print $sock "set $key 0 0 $biglen\r\n$big\r\n"; is(scalar <$sock>, "STORED\r\n", "stored big"); my ($f, $v, $c) = $mc->get($key); } } # diag "Test stats settings." { my %stats = $mc->stats('settings'); is(1024, $stats{'maxconns'}); # we run SSL tests over TCP; hence the domain_socket # is expected to be NULL. if (enabled_tls_testing()) { is('NULL', $stats{'domain_socket'}); } else { isnt('NULL', $stats{'domain_socket'}); } is('on', $stats{'evictions'}); is('yes', $stats{'cas_enabled'}); is('yes', $stats{'flush_enabled'}); } # diag "Test quit commands."; { my $s2 = new_memcached(); my $mc2 = MC::Client->new($s2); $mc2->send_command(CMD_QUITQ, '', '', 0, '', 0); # Five seconds ought to be enough to get hung up on. my $oldalarmt = alarm(5); # Verify we can't read anything. my $bytesread = -1; eval { local $SIG{'ALRM'} = sub { die "timeout" }; my $data = ""; $bytesread = sysread($mc2->{socket}, $data, 24), }; is($bytesread, 0, "Read after quit."); # Restore signal stuff. alarm($oldalarmt); } # diag "Test protocol boundary overruns"; { use List::Util qw[min]; # Attempting some protocol overruns by toying around with the edge # of the data buffer at a few different sizes. This assumes the # boundary is at or around 2048 bytes. for (my $i = 1900; $i < 2100; $i++) { my $k = "test_key_$i"; my $v = 'x' x $i; # diag "Trying $i $k"; my $extra = pack "NN", 82, 0; my $data = $mc->build_command(::CMD_SETQ, $k, $v, 0, $extra, 0); $data .= $mc->build_command(::CMD_SETQ, "alt_$k", "blah", 0, $extra, 0); if (length($data) > 2024) { for (my $j = 2024; $j < min(2096, length($data)); $j++) { $mc->{socket}->syswrite(substr($data, 0, $j)); $mc->flush_socket; sleep(0.001); $mc->{socket}->syswrite(substr($data, $j)); $mc->flush_socket; } } else { $mc->{socket}->syswrite($data); } $mc->flush_socket; $check->($k, 82, $v); $check->("alt_$k", 82, "blah"); } } # Along with the assertion added to the code to verify we're staying # within bounds when we do a stats detail dump (detail turned on at # the top). my %stats = $mc->stats('detail dump'); # This test causes a disconnection. { # diag "Key too large."; my $key = "x" x 365; eval { $mc->get($key, 'should die', 10, 10); }; ok($@->einval, "Invalid key length"); } done_testing(); # ###################################################################### # Test ends around here. # ###################################################################### package MC::Client; use strict; use warnings; use fields qw(socket); use IO::Socket::INET; sub new { my $self = shift; my ($s) = @_; $s = $server unless defined $s; my $sock = $s->sock; $self = fields::new($self); $self->{socket} = $sock; return $self; } sub build_command { my $self = shift; die "Not enough args to send_command" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; $extra_header = '' unless defined $extra_header; my $keylen = length($key); my $vallen = length($val); my $extralen = length($extra_header); my $datatype = 0; # field for future use my $reserved = 0; # field for future use my $totallen = $keylen + $vallen + $extralen; my $ident_hi = 0; my $ident_lo = 0; if ($cas) { $ident_hi = int($cas / 2 ** 32); $ident_lo = int($cas % 2 ** 32); } my $msg = pack(::REQ_PKT_FMT, ::REQ_MAGIC, $cmd, $keylen, $extralen, $datatype, $reserved, $totallen, $opaque, $ident_hi, $ident_lo); my $full_msg = $msg . $extra_header . $key . $val; return $full_msg; } sub send_command { my $self = shift; die "Not enough args to send_command" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; my $full_msg = $self->build_command($cmd, $key, $val, $opaque, $extra_header, $cas); my $sent = 0; my $data_len = length($full_msg); while ($sent < $data_len) { my $sent_bytes = $self->{socket}->syswrite($full_msg, $data_len - $sent > MemcachedTest::MAX_READ_WRITE_SIZE ? MemcachedTest::MAX_READ_WRITE_SIZE : ($data_len - $sent), $sent); last if ($sent_bytes <= 0); $sent += $sent_bytes; } die("Send failed: $!") unless $data_len; if($sent != $data_len) { die("only sent $sent of " . length($full_msg) . " bytes"); } } sub flush_socket { my $self = shift; $self->{socket}->flush; } # Send a silent command and ensure it doesn't respond. sub send_silent { my $self = shift; die "Not enough args to send_silent" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; $self->send_command($cmd, $key, $val, $opaque, $extra_header, $cas); $self->send_command(::CMD_NOOP, '', '', $opaque + 1); my ($ropaque, $data) = $self->_handle_single_response; Test::More::is($ropaque, $opaque + 1); } sub silent_mutation { my $self = shift; my ($cmd, $key, $value) = @_; $empty->($key); my $extra = pack "NN", 82, 0; $mc->send_silent($cmd, $key, $value, 7278552, $extra, 0); $check->($key, 82, $value); } sub _handle_single_response { my $self = shift; my $myopaque = shift; my $hdr = ""; while(::MIN_RECV_BYTES - length($hdr) > 0) { $self->{socket}->sysread(my $response, ::MIN_RECV_BYTES - length($hdr)); $hdr .= $response; } Test::More::is(length($hdr), ::MIN_RECV_BYTES, "Expected read length"); my ($magic, $cmd, $keylen, $extralen, $datatype, $status, $remaining, $opaque, $ident_hi, $ident_lo) = unpack(::RES_PKT_FMT, $hdr); Test::More::is($magic, ::RES_MAGIC, "Got proper response magic"); my $cas = ($ident_hi * 2 ** 32) + $ident_lo; return ($opaque, '', $cas, 0) if($remaining == 0); # fetch the value my $rv=""; while($remaining - length($rv) > 0) { $self->{socket}->sysread(my $buf, $remaining - length($rv)); $rv .= $buf; } if(length($rv) != $remaining) { my $found = length($rv); die("Expected $remaining bytes, got $found"); } if (defined $myopaque) { Test::More::is($opaque, $myopaque, "Expected opaque"); } else { Test::More::pass("Implicit pass since myopaque is undefined"); } if ($status) { die MC::Error->new($status, $rv); } return ($opaque, $rv, $cas, $keylen); } sub _do_command { my $self = shift; die unless @_ >= 3; my ($cmd, $key, $val, $extra_header, $cas) = @_; $extra_header = '' unless defined $extra_header; my $opaque = int(rand(2**32)); $self->send_command($cmd, $key, $val, $opaque, $extra_header, $cas); my (undef, $rv, $rcas) = $self->_handle_single_response($opaque); return ($rv, $rcas); } sub _incrdecr_header { my $self = shift; my ($amt, $init, $exp) = @_; my $amt_hi = int($amt / 2 ** 32); my $amt_lo = int($amt % 2 ** 32); my $init_hi = int($init / 2 ** 32); my $init_lo = int($init % 2 ** 32); my $extra_header = pack(::INCRDECR_PKT_FMT, $amt_hi, $amt_lo, $init_hi, $init_lo, $exp); return $extra_header; } sub _incrdecr_cas { my $self = shift; my ($cmd, $key, $amt, $init, $exp) = @_; my ($data, $rcas) = $self->_do_command($cmd, $key, '', $self->_incrdecr_header($amt, $init, $exp)); my $header = substr $data, 0, 8, ''; my ($resp_hi, $resp_lo) = unpack "NN", $header; my $resp = ($resp_hi * 2 ** 32) + $resp_lo; return $resp, $rcas; } sub _incrdecr { my $self = shift; my ($v, $c) = $self->_incrdecr_cas(@_); return $v } sub silent_incrdecr { my $self = shift; my ($cmd, $key, $amt, $init, $exp) = @_; my $opaque = 8275753; $mc->send_silent($cmd, $key, '', $opaque, $mc->_incrdecr_header($amt, $init, $exp)); } sub stats { my $self = shift; my $key = shift; my $cas = 0; my $opaque = int(rand(2**32)); $self->send_command(::CMD_STAT, $key, '', $opaque, '', $cas); my %rv = (); my $found_key = ''; my $found_val = ''; do { my ($op, $data, $cas, $keylen) = $self->_handle_single_response($opaque); if($keylen > 0) { $found_key = substr($data, 0, $keylen); $found_val = substr($data, $keylen); $rv{$found_key} = $found_val; } else { $found_key = ''; } } while($found_key ne ''); return %rv; } sub get { my $self = shift; my $key = shift; my ($rv, $cas) = $self->_do_command(::CMD_GET, $key, '', ''); my $header = substr $rv, 0, 4, ''; my $flags = unpack("N", $header); return ($flags, $rv, $cas); } sub get_multi { my $self = shift; my @keys = @_; for (my $i = 0; $i < @keys; $i++) { $self->send_command(::CMD_GETQ, $keys[$i], '', $i, '', 0); } my $terminal = @keys + 10; $self->send_command(::CMD_NOOP, '', '', $terminal); my %return; while (1) { my ($opaque, $data) = $self->_handle_single_response; last if $opaque == $terminal; my $header = substr $data, 0, 4, ''; my $flags = unpack("N", $header); $return{$keys[$opaque]} = [$flags, $data]; } return %return if wantarray; return \%return; } sub touch { my $self = shift; my ($key, $expire) = @_; my $extra_header = pack "N", $expire; my $cas = 0; return $self->_do_command(::CMD_TOUCH, $key, '', $extra_header, $cas); } sub gat { my $self = shift; my $key = shift; my $expire = shift; my $extra_header = pack "N", $expire; my ($rv, $cas) = $self->_do_command(::CMD_GAT, $key, '', $extra_header); my $header = substr $rv, 0, 4, ''; my $flags = unpack("N", $header); return ($flags, $rv, $cas); } sub version { my $self = shift; return $self->_do_command(::CMD_VERSION, '', ''); } sub flush { my $self = shift; return $self->_do_command(::CMD_FLUSH, '', ''); } sub add { my $self = shift; my ($key, $val, $flags, $expire) = @_; my $extra_header = pack "NN", $flags, $expire; my $cas = 0; return $self->_do_command(::CMD_ADD, $key, $val, $extra_header, $cas); } sub set { my $self = shift; my ($key, $val, $flags, $expire, $cas) = @_; my $extra_header = pack "NN", $flags, $expire; return $self->_do_command(::CMD_SET, $key, $val, $extra_header, $cas); } sub _append_prepend { my $self = shift; my ($cmd, $key, $val, $cas) = @_; return $self->_do_command($cmd, $key, $val, '', $cas); } sub replace { my $self = shift; my ($key, $val, $flags, $expire) = @_; my $extra_header = pack "NN", $flags, $expire; my $cas = 0; return $self->_do_command(::CMD_REPLACE, $key, $val, $extra_header, $cas); } sub delete { my $self = shift; my ($key) = @_; return $self->_do_command(::CMD_DELETE, $key, ''); } sub incr { my $self = shift; my ($key, $amt, $init, $exp) = @_; $amt = 1 unless defined $amt; $init = 0 unless defined $init; $exp = 0 unless defined $exp; return $self->_incrdecr(::CMD_INCR, $key, $amt, $init, $exp); } sub incr_cas { my $self = shift; my ($key, $amt, $init, $exp) = @_; $amt = 1 unless defined $amt; $init = 0 unless defined $init; $exp = 0 unless defined $exp; return $self->_incrdecr_cas(::CMD_INCR, $key, $amt, $init, $exp); } sub decr { my $self = shift; my ($key, $amt, $init, $exp) = @_; $amt = 1 unless defined $amt; $init = 0 unless defined $init; $exp = 0 unless defined $exp; return $self->_incrdecr(::CMD_DECR, $key, $amt, $init, $exp); } sub noop { my $self = shift; return $self->_do_command(::CMD_NOOP, '', ''); } package MC::Error; use strict; use warnings; use constant ERR_UNKNOWN_CMD => 0x81; use constant ERR_NOT_FOUND => 0x1; use constant ERR_EXISTS => 0x2; use constant ERR_TOO_BIG => 0x3; use constant ERR_EINVAL => 0x4; use constant ERR_NOT_STORED => 0x5; use constant ERR_DELTA_BADVAL => 0x6; use overload '""' => sub { my $self = shift; return "Memcache Error ($self->[0]): $self->[1]"; }; sub new { my $class = shift; my $error = [@_]; my $self = bless $error, (ref $class || $class); return $self; } sub not_found { my $self = shift; return $self->[0] == ERR_NOT_FOUND; } sub exists { my $self = shift; return $self->[0] == ERR_EXISTS; } sub too_big { my $self = shift; return $self->[0] == ERR_TOO_BIG; } sub delta_badval { my $self = shift; return $self->[0] == ERR_DELTA_BADVAL; } sub einval { my $self = shift; return $self->[0] == ERR_EINVAL; } # vim: filetype=perl memcached-1.5.22/t/64bit.t0000755000175000017500000000216212416643766012033 00000000000000#!/usr/bin/perl use strict; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; $ENV{T_MEMD_INITIAL_MALLOC} = "4294967328"; # 2**32 + 32 , just over 4GB $ENV{T_MEMD_SLABS_ALLOC} = 0; # don't preallocate slabs my $server = new_memcached("-m 4098 -M"); my $sock = $server->sock; my ($stats, $slabs) = @_; $stats = mem_stats($sock); if ($stats->{'pointer_size'} eq "32") { plan skip_all => 'Skipping 64-bit tests on 32-bit build'; exit 0; } else { plan tests => 6; } is($stats->{'pointer_size'}, 64, "is 64 bit"); is($stats->{'limit_maxbytes'}, "4297064448", "max bytes is 4098 MB"); $slabs = mem_stats($sock, 'slabs'); is($slabs->{'total_malloced'}, "4294967328", "expected (faked) value of total_malloced"); is($slabs->{'active_slabs'}, 0, "no active slabs"); my $hit_limit = 0; for (1..5) { my $size = 400 * 1024; my $data = "a" x $size; print $sock "set big$_ 0 0 $size\r\n$data\r\n"; my $res = <$sock>; $hit_limit = 1 if $res ne "STORED\r\n"; } ok($hit_limit, "hit size limit"); $slabs = mem_stats($sock, 'slabs'); is($slabs->{'active_slabs'}, 1, "1 active slab"); memcached-1.5.22/t/issue_183.t0000664000175000017500000000122313461170555012612 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 5; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached("-o no_modern"); my $sock = $server->sock; print $sock "set key 0 0 1\r\n1\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key"); my $s1 = mem_stats($sock); my $r1 = $s1->{"reclaimed"}; is ($r1, "0", "Objects should not be reclaimed"); sleep(2); print $sock "flush_all\r\n"; is (scalar <$sock>, "OK\r\n", "Cache flushed"); print $sock "set key 0 0 1\r\n1\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key"); my $s2 = mem_stats($sock); my $r2 = $s2->{"reclaimed"}; is ($r2, "1", "Objects should be reclaimed"); memcached-1.5.22/t/watcher_connid.t0000664000175000017500000000377513575022500014064 00000000000000#!/usr/bin/perl # Test for adding connection id to the output when watching fetchers # and mutations. # Note that this test relies on the order of connection establishments. This # could be improved if there's a way for a client to retrieve its connection id. use strict; use warnings; use Socket qw/SO_RCVBUF/; use Test::More tests => 4; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached('-m 60 -o watcher_logbuf_size=8'); my $client_first = $server->sock; my $stats; # get the first client's connection id print $client_first "stats conns\r\n"; while (<$client_first>) { last if /^(\.|END)/; $stats = $_; } my $cfd_first =(split(':', $stats))[0]; $cfd_first =~ s/[^0-9]//g; # start watching fetchers and mutations my $watcher = $server->new_sock; print $watcher "watch fetchers mutations\n"; my $res = <$watcher>; is($res, "OK\r\n", "watching enabled for fetchers and mutations"); # first client does a set, which will result in a get and a set print $client_first "set foo 0 0 5 noreply\r\nhello\r\n"; # ensure client's connection id is correct $res = <$watcher>; print $res; like($res, qr/ts=\d+\.\d+\ gid=\d+ type=item_get key=foo status=not_found clsid=\d+ cfd=$cfd_first/, "Saw a miss with the connection id $cfd_first"); $res = <$watcher>; print $res; like($res, qr/ts=\d+\.\d+\ gid=\d+ type=item_store key=foo status=stored cmd=set ttl=\d+ clsid=\d+ cfd=$cfd_first/, "Saw a set with the connection id $cfd_first"); # get the second client's connection id my $client_second = $server->new_sock; print $client_second "stats conns\r\n"; while (<$client_second>) { last if /^(\.|END)/; $stats = $_; } my $cfd_second =(split(':', $stats))[0]; $cfd_second =~ s/[^0-9]//g; # second client does a get print $client_second "get foo\r\n"; # now we should see second client's connection id $res = <$watcher>; print $res; like($res, qr/ts=\d+\.\d+\ gid=\d+ type=item_get key=foo status=found clsid=\d+ cfd=$cfd_second/, "Saw a get with the connection id $cfd_second"); memcached-1.5.22/t/evictions.t0000664000175000017500000000155113250303500013056 00000000000000#!/usr/bin/perl # Test the 'stats items' evictions counters. use strict; use Test::More tests => 92; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached("-m 3"); my $sock = $server->sock; my $value = "B"x66560; my $key = 0; # These aren't set to expire. for ($key = 0; $key < 40; $key++) { print $sock "set key$key 0 0 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key$key"); } # These ones would expire in 600 seconds. for ($key = 0; $key < 50; $key++) { print $sock "set key$key 0 600 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key$key"); } my $stats = mem_stats($sock, "items"); my $evicted = $stats->{"items:31:evicted"}; isnt($evicted, "0", "check evicted"); my $evicted_nonzero = $stats->{"items:31:evicted_nonzero"}; isnt($evicted_nonzero, "0", "check evicted_nonzero"); memcached-1.5.22/t/binary-extstore.t0000775000175000017500000004076713550052525014244 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use Data::Dumper qw/Dumper/; my $ext_path; if (!supports_extstore()) { plan skip_all => 'extstore not enabled'; exit 0; } $ext_path = "/tmp/extstore.$$"; my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,no_lru_crawler,slab_automove=0"); ok($server, "started the server"); # Based almost 100% off testClient.py which is: # Copyright (c) 2007 Dustin Sallings # Command constants use constant CMD_GET => 0x00; use constant CMD_SET => 0x01; use constant CMD_ADD => 0x02; use constant CMD_REPLACE => 0x03; use constant CMD_DELETE => 0x04; use constant CMD_INCR => 0x05; use constant CMD_DECR => 0x06; use constant CMD_QUIT => 0x07; use constant CMD_FLUSH => 0x08; use constant CMD_GETQ => 0x09; use constant CMD_NOOP => 0x0A; use constant CMD_VERSION => 0x0B; use constant CMD_GETK => 0x0C; use constant CMD_GETKQ => 0x0D; use constant CMD_APPEND => 0x0E; use constant CMD_PREPEND => 0x0F; use constant CMD_STAT => 0x10; use constant CMD_SETQ => 0x11; use constant CMD_ADDQ => 0x12; use constant CMD_REPLACEQ => 0x13; use constant CMD_DELETEQ => 0x14; use constant CMD_INCREMENTQ => 0x15; use constant CMD_DECREMENTQ => 0x16; use constant CMD_QUITQ => 0x17; use constant CMD_FLUSHQ => 0x18; use constant CMD_APPENDQ => 0x19; use constant CMD_PREPENDQ => 0x1A; use constant CMD_TOUCH => 0x1C; use constant CMD_GAT => 0x1D; use constant CMD_GATQ => 0x1E; use constant CMD_GATK => 0x23; use constant CMD_GATKQ => 0x24; # REQ and RES formats are divided even though they currently share # the same format, since they _could_ differ in the future. use constant REQ_PKT_FMT => "CCnCCnNNNN"; use constant RES_PKT_FMT => "CCnCCnNNNN"; use constant INCRDECR_PKT_FMT => "NNNNN"; use constant MIN_RECV_BYTES => length(pack(RES_PKT_FMT)); use constant REQ_MAGIC => 0x80; use constant RES_MAGIC => 0x81; my $mc = MC::Client->new; my $check = sub { my ($key, $orig_flags, $orig_val) = @_; my ($flags, $val, $cas) = $mc->get($key); is($flags, $orig_flags, "Flags is set properly"); ok($val eq $orig_val || $val == $orig_val, $val . " = " . $orig_val); }; my $set = sub { my ($key, $exp, $orig_flags, $orig_value) = @_; $mc->set($key, $orig_value, $orig_flags, $exp); $check->($key, $orig_flags, $orig_value); }; my $empty = sub { my $key = shift; my $rv =()= eval { $mc->get($key) }; is($rv, 0, "Didn't get a result from get"); ok($@->not_found, "We got a not found error when we expected one"); }; my $delete = sub { my ($key, $when) = @_; $mc->delete($key, $when); $empty->($key); }; my $value; my $bigvalue; { my @chars = ("C".."Z"); for (1 .. 20000) { $value .= $chars[rand @chars]; } for (1 .. 800000) { $bigvalue .= $chars[rand @chars]; } } # diag "small object"; $set->('x', 10, 19, "somevalue"); # check extstore counters { my %stats = $mc->stats(''); is($stats{extstore_objects_written}, 0); } # diag "Delete"; #$delete->('x'); # diag "Flush"; #$empty->('y'); # fill some larger objects { my $keycount = 1000; for (1 .. $keycount) { $set->("nfoo$_", 0, 19, $value); } # wait for a flush sleep 4; # value returns for one flushed object. $check->('nfoo1', 19, $value); # check extstore counters my %stats = $mc->stats(''); cmp_ok($stats{extstore_page_allocs}, '>', 0, 'at least one page allocated'); cmp_ok($stats{extstore_objects_written}, '>', $keycount / 2, 'some objects written'); cmp_ok($stats{extstore_bytes_written}, '>', length($value) * 2, 'some bytes written'); cmp_ok($stats{get_extstore}, '>', 0, 'one object was fetched'); cmp_ok($stats{extstore_objects_read}, '>', 0, 'one object read'); cmp_ok($stats{extstore_bytes_read}, '>', length($value), 'some bytes read'); # Test multiget my $rv = $mc->get_multi(qw(nfoo2 nfoo3 noexist)); is($rv->{nfoo2}->[1], $value, 'multiget nfoo2'); is($rv->{nfoo3}->[1], $value, 'multiget nfoo2'); # Remove half of the keys for the next test. for (1 .. $keycount) { next unless $_ % 2 == 0; $delete->("nfoo$_"); } my %stats2 = $mc->stats(''); cmp_ok($stats{extstore_bytes_used}, '>', $stats2{extstore_bytes_used}, 'bytes used dropped after deletions'); cmp_ok($stats{extstore_objects_used}, '>', $stats2{extstore_objects_used}, 'objects used dropped after deletions'); is($stats2{badcrc_from_extstore}, 0, 'CRC checks successful'); # delete the rest for (1 .. $keycount) { next unless $_ % 2 == 1; $delete->("nfoo$_"); } } # check evictions and misses { my $keycount = 1000; for (1 .. $keycount) { $set->("mfoo$_", 0, 19, $value); } sleep 4; for ($keycount .. ($keycount*3)) { $set->("mfoo$_", 0, 19, $value); } sleep 4; # FIXME: Need to sample through a few values, or fix eviction to be # more accurate. On 32bit systems some pages unused to this point get # filled after the first few items, then the eviction algo pulls those # pages since they have the lowest version number, leaving older objects # in memory and evicting newer ones. for (1 .. ($keycount*3)) { next unless $_ % 100 == 0; eval { $mc->get("mfoo$_"); }; } my %s = $mc->stats(''); cmp_ok($s{extstore_objects_evicted}, '>', 0); cmp_ok($s{miss_from_extstore}, '>', 0); } # store and re-fetch a chunked value { my %stats = $mc->stats(''); $set->("bigvalue", 0, 0, $bigvalue); sleep 4; $check->("bigvalue", 0, $bigvalue); my %stats2 = $mc->stats(''); cmp_ok($stats2{extstore_objects_written}, '>', $stats{extstore_objects_written}, "a large value flushed"); } # ensure ASCII can still fetch the chunked value. { my $ns = $server->new_sock; my %s1 = $mc->stats(''); mem_get_is($ns, "bigvalue", $bigvalue); print $ns "extstore recache_rate 1\r\n"; is(scalar <$ns>, "OK\r\n", "recache rate upped"); for (1..3) { mem_get_is($ns, "bigvalue", $bigvalue); $check->('bigvalue', 0, $bigvalue); } my %s2 = $mc->stats(''); cmp_ok($s2{recache_from_extstore}, '>', $s1{recache_from_extstore}, 'a new recache happened'); } done_testing(); END { unlink $ext_path if $ext_path; } # ###################################################################### # Test ends around here. # ###################################################################### package MC::Client; use strict; use warnings; use fields qw(socket); use IO::Socket::INET; sub new { my $self = shift; my ($s) = @_; $s = $server unless defined $s; my $sock = $s->sock; $self = fields::new($self); $self->{socket} = $sock; return $self; } sub build_command { my $self = shift; die "Not enough args to send_command" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; $extra_header = '' unless defined $extra_header; my $keylen = length($key); my $vallen = length($val); my $extralen = length($extra_header); my $datatype = 0; # field for future use my $reserved = 0; # field for future use my $totallen = $keylen + $vallen + $extralen; my $ident_hi = 0; my $ident_lo = 0; if ($cas) { $ident_hi = int($cas / 2 ** 32); $ident_lo = int($cas % 2 ** 32); } my $msg = pack(::REQ_PKT_FMT, ::REQ_MAGIC, $cmd, $keylen, $extralen, $datatype, $reserved, $totallen, $opaque, $ident_hi, $ident_lo); my $full_msg = $msg . $extra_header . $key . $val; return $full_msg; } sub send_command { my $self = shift; die "Not enough args to send_command" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; my $full_msg = $self->build_command($cmd, $key, $val, $opaque, $extra_header, $cas); my $sent = $self->{socket}->send($full_msg); die("Send failed: $!") unless $sent; if($sent != length($full_msg)) { die("only sent $sent of " . length($full_msg) . " bytes"); } } sub flush_socket { my $self = shift; $self->{socket}->flush; } # Send a silent command and ensure it doesn't respond. sub send_silent { my $self = shift; die "Not enough args to send_silent" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; $self->send_command($cmd, $key, $val, $opaque, $extra_header, $cas); $self->send_command(::CMD_NOOP, '', '', $opaque + 1); my ($ropaque, $data) = $self->_handle_single_response; Test::More::is($ropaque, $opaque + 1); } sub silent_mutation { my $self = shift; my ($cmd, $key, $value) = @_; $empty->($key); my $extra = pack "NN", 82, 0; $mc->send_silent($cmd, $key, $value, 7278552, $extra, 0); $check->($key, 82, $value); } sub _handle_single_response { my $self = shift; my $myopaque = shift; my $hdr = ""; while(::MIN_RECV_BYTES - length($hdr) > 0) { $self->{socket}->recv(my $response, ::MIN_RECV_BYTES - length($hdr)); $hdr .= $response; } Test::More::is(length($hdr), ::MIN_RECV_BYTES, "Expected read length"); my ($magic, $cmd, $keylen, $extralen, $datatype, $status, $remaining, $opaque, $ident_hi, $ident_lo) = unpack(::RES_PKT_FMT, $hdr); Test::More::is($magic, ::RES_MAGIC, "Got proper response magic"); my $cas = ($ident_hi * 2 ** 32) + $ident_lo; return ($opaque, '', $cas, 0) if($remaining == 0); # fetch the value my $rv=""; while($remaining - length($rv) > 0) { $self->{socket}->recv(my $buf, $remaining - length($rv)); $rv .= $buf; } if(length($rv) != $remaining) { my $found = length($rv); die("Expected $remaining bytes, got $found"); } if (defined $myopaque) { Test::More::is($opaque, $myopaque, "Expected opaque"); } else { Test::More::pass("Implicit pass since myopaque is undefined"); } if ($status) { die MC::Error->new($status, $rv); } return ($opaque, $rv, $cas, $keylen); } sub _do_command { my $self = shift; die unless @_ >= 3; my ($cmd, $key, $val, $extra_header, $cas) = @_; $extra_header = '' unless defined $extra_header; my $opaque = int(rand(2**32)); $self->send_command($cmd, $key, $val, $opaque, $extra_header, $cas); my (undef, $rv, $rcas) = $self->_handle_single_response($opaque); return ($rv, $rcas); } sub _incrdecr_header { my $self = shift; my ($amt, $init, $exp) = @_; my $amt_hi = int($amt / 2 ** 32); my $amt_lo = int($amt % 2 ** 32); my $init_hi = int($init / 2 ** 32); my $init_lo = int($init % 2 ** 32); my $extra_header = pack(::INCRDECR_PKT_FMT, $amt_hi, $amt_lo, $init_hi, $init_lo, $exp); return $extra_header; } sub _incrdecr_cas { my $self = shift; my ($cmd, $key, $amt, $init, $exp) = @_; my ($data, $rcas) = $self->_do_command($cmd, $key, '', $self->_incrdecr_header($amt, $init, $exp)); my $header = substr $data, 0, 8, ''; my ($resp_hi, $resp_lo) = unpack "NN", $header; my $resp = ($resp_hi * 2 ** 32) + $resp_lo; return $resp, $rcas; } sub _incrdecr { my $self = shift; my ($v, $c) = $self->_incrdecr_cas(@_); return $v } sub silent_incrdecr { my $self = shift; my ($cmd, $key, $amt, $init, $exp) = @_; my $opaque = 8275753; $mc->send_silent($cmd, $key, '', $opaque, $mc->_incrdecr_header($amt, $init, $exp)); } sub stats { my $self = shift; my $key = shift; my $cas = 0; my $opaque = int(rand(2**32)); $self->send_command(::CMD_STAT, $key, '', $opaque, '', $cas); my %rv = (); my $found_key = ''; my $found_val = ''; do { my ($op, $data, $cas, $keylen) = $self->_handle_single_response($opaque); if($keylen > 0) { $found_key = substr($data, 0, $keylen); $found_val = substr($data, $keylen); $rv{$found_key} = $found_val; } else { $found_key = ''; } } while($found_key ne ''); return %rv; } sub get { my $self = shift; my $key = shift; my ($rv, $cas) = $self->_do_command(::CMD_GET, $key, '', ''); my $header = substr $rv, 0, 4, ''; my $flags = unpack("N", $header); return ($flags, $rv, $cas); } sub get_multi { my $self = shift; my @keys = @_; for (my $i = 0; $i < @keys; $i++) { $self->send_command(::CMD_GETQ, $keys[$i], '', $i, '', 0); } my $terminal = @keys + 10; $self->send_command(::CMD_NOOP, '', '', $terminal); my %return; while (1) { my ($opaque, $data) = $self->_handle_single_response; last if $opaque == $terminal; my $header = substr $data, 0, 4, ''; my $flags = unpack("N", $header); $return{$keys[$opaque]} = [$flags, $data]; } return %return if wantarray; return \%return; } sub touch { my $self = shift; my ($key, $expire) = @_; my $extra_header = pack "N", $expire; my $cas = 0; return $self->_do_command(::CMD_TOUCH, $key, '', $extra_header, $cas); } sub gat { my $self = shift; my $key = shift; my $expire = shift; my $extra_header = pack "N", $expire; my ($rv, $cas) = $self->_do_command(::CMD_GAT, $key, '', $extra_header); my $header = substr $rv, 0, 4, ''; my $flags = unpack("N", $header); return ($flags, $rv, $cas); } sub version { my $self = shift; return $self->_do_command(::CMD_VERSION, '', ''); } sub flush { my $self = shift; return $self->_do_command(::CMD_FLUSH, '', ''); } sub add { my $self = shift; my ($key, $val, $flags, $expire) = @_; my $extra_header = pack "NN", $flags, $expire; my $cas = 0; return $self->_do_command(::CMD_ADD, $key, $val, $extra_header, $cas); } sub set { my $self = shift; my ($key, $val, $flags, $expire, $cas) = @_; my $extra_header = pack "NN", $flags, $expire; return $self->_do_command(::CMD_SET, $key, $val, $extra_header, $cas); } sub _append_prepend { my $self = shift; my ($cmd, $key, $val, $cas) = @_; return $self->_do_command($cmd, $key, $val, '', $cas); } sub replace { my $self = shift; my ($key, $val, $flags, $expire) = @_; my $extra_header = pack "NN", $flags, $expire; my $cas = 0; return $self->_do_command(::CMD_REPLACE, $key, $val, $extra_header, $cas); } sub delete { my $self = shift; my ($key) = @_; return $self->_do_command(::CMD_DELETE, $key, ''); } sub incr { my $self = shift; my ($key, $amt, $init, $exp) = @_; $amt = 1 unless defined $amt; $init = 0 unless defined $init; $exp = 0 unless defined $exp; return $self->_incrdecr(::CMD_INCR, $key, $amt, $init, $exp); } sub incr_cas { my $self = shift; my ($key, $amt, $init, $exp) = @_; $amt = 1 unless defined $amt; $init = 0 unless defined $init; $exp = 0 unless defined $exp; return $self->_incrdecr_cas(::CMD_INCR, $key, $amt, $init, $exp); } sub decr { my $self = shift; my ($key, $amt, $init, $exp) = @_; $amt = 1 unless defined $amt; $init = 0 unless defined $init; $exp = 0 unless defined $exp; return $self->_incrdecr(::CMD_DECR, $key, $amt, $init, $exp); } sub noop { my $self = shift; return $self->_do_command(::CMD_NOOP, '', ''); } package MC::Error; use strict; use warnings; use constant ERR_UNKNOWN_CMD => 0x81; use constant ERR_NOT_FOUND => 0x1; use constant ERR_EXISTS => 0x2; use constant ERR_TOO_BIG => 0x3; use constant ERR_EINVAL => 0x4; use constant ERR_NOT_STORED => 0x5; use constant ERR_DELTA_BADVAL => 0x6; use overload '""' => sub { my $self = shift; return "Memcache Error ($self->[0]): $self->[1]"; }; sub new { my $class = shift; my $error = [@_]; my $self = bless $error, (ref $class || $class); return $self; } sub not_found { my $self = shift; return $self->[0] == ERR_NOT_FOUND; } sub exists { my $self = shift; return $self->[0] == ERR_EXISTS; } sub too_big { my $self = shift; return $self->[0] == ERR_TOO_BIG; } sub delta_badval { my $self = shift; return $self->[0] == ERR_DELTA_BADVAL; } sub einval { my $self = shift; return $self->[0] == ERR_EINVAL; } # vim: filetype=perl memcached-1.5.22/t/unixsocket.t0000755000175000017500000000076712416643766013310 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 3; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $filename = "/tmp/memcachetest$$"; my $server = new_memcached("-s $filename"); my $sock = $server->sock; ok(-S $filename, "creating unix domain socket $filename"); # set foo (and should get it) print $sock "set foo 0 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); unlink($filename); ## Just some basic stuff for now... memcached-1.5.22/t/maxconns.t0000775000175000017500000000111213461170555012715 00000000000000#!/usr/bin/perl # NOTE: This test never worked. Memcached would ignore maxconns requests lower # than the current ulimit. Test needs to be updated. use strict; use warnings; use Test::More tests => 11; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; # start up a server with 10 maximum connections my $server = new_memcached('-c 100'); my $sock = $server->sock; my @sockets; ok(defined($sock), 'Connection 0'); push (@sockets, $sock); foreach my $conn (1..10) { $sock = $server->new_sock; ok(defined($sock), "Made connection $conn"); push(@sockets, $sock); } memcached-1.5.22/t/issue_152.t0000664000175000017500000000060613250303500012572 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 2; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $key = "a"x251; print $sock "set a 1 0 1\r\na\r\n"; is (scalar <$sock>, "STORED\r\n", "Stored key"); print $sock "get a $key\r\n"; is (scalar <$sock>, "CLIENT_ERROR bad command line format\r\n", "illegal key"); memcached-1.5.22/t/line-lengths.t0000775000175000017500000000100413250303500013440 00000000000000#!/usr/bin/perl use strict; use FindBin qw($Bin); our @files; BEGIN { chdir "$Bin/.." or die; @files = ( "doc/protocol.txt" ); } use Test::More tests => scalar(@files); foreach my $f (@files) { open(my $fh, $f) or die("Can't open $f"); my @long_lines = (); my $line_number = 0; while(<$fh>) { $line_number++; if(length($_) > 80) { push(@long_lines, $line_number); } } close($fh); ok(@long_lines == 0, "$f has a long lines: @long_lines"); } memcached-1.5.22/t/issue_70.t0000664000175000017500000000114413461170555012527 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 4; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; print $sock "set issue70 0 0 0\r\n\r\n"; is (scalar <$sock>, "STORED\r\n", "stored issue70"); print $sock "set issue70 0 0 -1\r\n"; is (scalar <$sock>, "CLIENT_ERROR bad command line format\r\n"); print $sock "set issue70 0 0 4294967295\r\n"; is (scalar <$sock>, "CLIENT_ERROR bad command line format\r\n"); print $sock "set issue70 0 0 2147483647\r\nscoobyscoobydoo"; is (scalar <$sock>, "CLIENT_ERROR bad command line format\r\n"); memcached-1.5.22/t/ssl_cert_refresh.t0000664000175000017500000000532213575022500014417 00000000000000#!/usr/bin/perl use strict; use warnings; use File::Copy; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; if (!enabled_tls_testing()) { plan skip_all => 'SSL testing is not enabled'; exit 0; } my $ca_cert = "t/" . MemcachedTest::CA_CRT; my $cert = "t/". MemcachedTest::SRV_CRT; my $key = "t/". MemcachedTest::SRV_KEY; my $ca_cert_back = "t/ca_cert_back"; my $cert_back = "t/cert_back"; my $key_back = "t/pkey_back"; my $new_cert_key = "t/server.pem"; my $default_crt_ou = "OU=Subunit of Test Organization"; my $server = new_memcached("-o ssl_ca_cert=$ca_cert"); my $stats = mem_stats($server->sock); my $pid = $stats->{pid}; my $sock = $server->sock; # This connection should return the default server certificate # memcached was started with. my $cert_details =$sock->dump_peer_certificate(); $cert_details =~ m/(OU=([^\/\n]*))/; is($1, $default_crt_ou, 'Got the default cert'); # Swap a new certificate with a key copy($ca_cert, $ca_cert_back) or die "CA cert backup failed: $!"; copy($cert, $cert_back) or die "Cert backup failed: $!"; copy($key, $key_back) or die "Key backup failed: $!"; copy($new_cert_key, $ca_cert) or die "New CA cert copy failed: $!"; copy($new_cert_key, $cert) or die "New Cert copy failed: $!"; copy($new_cert_key, $key) or die "New key copy failed: $!"; # Ask server to refresh certificates print $sock "refresh_certs\r\n"; is(scalar <$sock>, "OK\r\n", "refreshed certificates"); # New connections should use the new certificate $cert_details = $server->new_sock->dump_peer_certificate(); $cert_details =~ m/(OU=([^\/]*))/; is($1, 'OU=FOR TESTING PURPOSES ONLY','Got the new cert'); # Old connection should use the previous certificate $cert_details = $sock->dump_peer_certificate(); $cert_details =~ m/(OU=([^\/\n]*))/; is($1, $default_crt_ou, 'Old connection still has the old cert'); # Just sleep a while to test the time_since_server_cert_refresh as it's counted # in seconds. sleep 2; $stats = mem_stats($sock); # Restore and ensure previous certificate is back for new connections. move($ca_cert_back, $ca_cert) or die "CA cert restore failed: $!"; move($cert_back, $cert) or die "Cert restore failed: $!"; move($key_back, $key) or die "Key restore failed: $!"; print $sock "refresh_certs\r\n"; is(scalar <$sock>, "OK\r\n", "refreshed certificates"); $cert_details = $server->new_sock->dump_peer_certificate(); $cert_details =~ m/(OU=([^\/\n]*))/; is($1, $default_crt_ou, 'Got the old cert back'); my $stats_after = mem_stats($sock); # We should see last refresh time is reset; hence the new # time_since_server_cert_refresh should be less. cmp_ok($stats_after->{time_since_server_cert_refresh}, '<', $stats->{time_since_server_cert_refresh}, 'Certs refreshed'); done_testing(); memcached-1.5.22/t/cakey.pem0000664000175000017500000000347613575022500012505 00000000000000-----BEGIN ENCRYPTED PRIVATE KEY----- MIIFHDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQIigxGWoxgfJACAggA MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECHgfMUpgAX2tBIIEyFcJT6kFw1TH kmwmbtB+MPW3+zUxoKZuP+L7UCaMHQPHcQ4/AThVNPPhXa++/lgL0P1q33SBhqsm HK5nRfzEe75bQIFoZ1FFg3YGaAB4KUPm5vJ4XhmyzCEiUvp75mgiw38fkL5ZCPqW eitNKgyAYEl9zIrF1cRyN/s6O6mPR8NOPbVL2ZtCGGvy/LN+mhlolvuMFySEEI+s nzu5682LU7QQOzLFB/q3Vqrt2tafzoIHYtj8YYVsPMO6bWkoKx1BWyTCuesFOLTN agnXx3AKtmD8Q0C3vvdmpqNnWw1iP1Vc4r/BZwmD2MFXUh8cExBGRaPIfUr41BoR p1bHqtcAsbipuU7qQ6wjnLyuqAvFQxy1LCBGNyBrzMMD25+lXh9BoeS9H5L+WW0N pcAPA5FoD1RQc27ZXVf0kRGEy8Vj/UsE3Pf74Z8u16bC+6ojAUvvvc49ERm0t85O T5rP9ql3sITtg4EvNdNhHtYdJGVkZQ7H8T9G1OXEf4dE/vqYNnqhtFm+5CyWb8Vk Rly2XFbBs4I4UnVIi0Nt0ybIqqwAvvKAU5kSOIEy6Q9mvUYyvBZrhY5Va2KEwaTU ig2yjWxWFoOArE1UVu2kJaslaOKqAr9OPqW510+6G9s+lREm7EYqUd6Ut1cwcXUv /s1fdx/As6U6qApH9+TC0XqNm5yXNCuADVtK2PmOoanczpEXZTZGijylGFDROQQj bofjoB0hiN80Xc+II0zClGwXwZz43Xy4uVIMbvMBi3yp03ct1RMwORPCGkiH/1uV iazsfV4DAD8KdGmgUL57WfnMPZJCEfqpM4YrM7rKu3Md8v8hKyFGUAdeNsCFTwwY g2E6NXWbtqGrUFC/r/70/axfijWRoyPaLcBoourc5HcZ+K/TmmY+uOxKBcJsfS/t HiRZ5l48sva+lZVWN6KNG5N93pa3TqgCCfp56VPOo3o3kj4XAJ3aMK6RsK3QrGi7 TTXlv/hKFYW3GUJ0IGNfinnARbdMoD6ww8nQLe3Id3HKzqh7xfQb4xJZpQhpSXVS KJmf5H80GHQnJq54Xwi7daYmxpTS+yNHuxKq+ryBqO7WeotNkOCbG6Flc/GBO6a3 7rbSOTmqpyJPTRPQEf1ogHNgCG9txPIbZRAi9PPJVn/gV/dKttUpj5OGI9fjDaSK ILEwLuokdRwTAGeFRIkucKnvA0pzXQkZRMG4D4kYzxskutUQVvKFAsqvEkb2c6qL j8MjKYfYZqxG44o68aOk8H42vR6lkemclj6byvLfnqbZPUka3MUnrG8oPI+vNLlg N57QpU2Kw5joBf1oCSROTXEv3BT/JYE3qQ2T6q/NGgG0s8aXbROh/7HaZJrENmoD pDJZp+bSGn6On6vmDu98F3fyubKgg611tA5pyDdWE4MSvJBGKHc2/HmqgxbLtK1c gn2BBefAi+Qqzc+XSeJxh2nlAx7ohvgEcodO3Vqb9eSoPviMHyO9bap8QIFJqTNW QK6rMqtou0QTEcXp5cIlVRW59zZHKEPpNpJ08fNtxB5n/Ngo+vo86LLVUysfbwkY p7GK4kraBGhU3rEXzuFBXn1Cr7015VrAC8mLafKqar7OLhLduHDRPxGidXJc6oxE 2PpJudZFhS6P+0M5AkMq2Q== -----END ENCRYPTED PRIVATE KEY----- memcached-1.5.22/t/issue_140.t0000664000175000017500000000171613461170555012612 00000000000000#!/usr/bin/perl use strict; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; plan skip_all => 'Fix for Issue 140 was only an illusion'; plan tests => 7; my $server = new_memcached(); my $sock = $server->sock; print $sock "set a 0 0 1\r\na\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key"); my $stats = mem_stats($sock, "items"); my $age = $stats->{"items:1:age"}; isnt ($age, "0", "Age should not be zero"); print $sock "flush_all\r\n"; is (scalar <$sock>, "OK\r\n", "items flushed"); my $stats = mem_stats($sock, "items"); my $age = $stats->{"items:1:age"}; is ($age, undef, "all should be gone"); print $sock "set a 0 1 1\r\na\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key"); my $stats = mem_stats($sock, "items"); my $age = $stats->{"items:1:age"}; isnt ($age, "0", "Age should not be zero"); sleep(3); my $stats = mem_stats($sock, "items"); my $age = $stats->{"items:1:age"}; is ($age, undef, "all should be gone"); memcached-1.5.22/t/binary-sasl.t0000775000175000017500000004264613615376577013351 00000000000000#!/usr/bin/perl use strict; use warnings; use Cwd; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $supports_sasl = supports_sasl(); use Test::More; if (supports_sasl()) { if ($ENV{'RUN_SASL_TESTS'}) { plan tests => 34; } else { plan skip_all => 'Skipping SASL tests'; exit 0; } } else { plan tests => 1; eval { my $server = new_memcached("-S"); }; ok($@, "Died with illegal -S args when SASL is not supported."); exit 0; } eval { my $server = new_memcached("-S -B auto"); }; ok($@, "SASL shouldn't be used with protocol auto negotiate"); eval { my $server = new_memcached("-S -B ascii"); }; ok($@, "SASL isn't implemented in the ascii protocol"); eval { my $server = new_memcached("-S -B binary -B ascii"); }; ok($@, "SASL isn't implemented in the ascii protocol"); # Based almost 100% off testClient.py which is: # Copyright (c) 2007 Dustin Sallings # Command constants use constant CMD_GET => 0x00; use constant CMD_SET => 0x01; use constant CMD_ADD => 0x02; use constant CMD_REPLACE => 0x03; use constant CMD_DELETE => 0x04; use constant CMD_INCR => 0x05; use constant CMD_DECR => 0x06; use constant CMD_QUIT => 0x07; use constant CMD_FLUSH => 0x08; use constant CMD_GETQ => 0x09; use constant CMD_NOOP => 0x0A; use constant CMD_VERSION => 0x0B; use constant CMD_GETK => 0x0C; use constant CMD_GETKQ => 0x0D; use constant CMD_APPEND => 0x0E; use constant CMD_PREPEND => 0x0F; use constant CMD_STAT => 0x10; use constant CMD_SETQ => 0x11; use constant CMD_ADDQ => 0x12; use constant CMD_REPLACEQ => 0x13; use constant CMD_DELETEQ => 0x14; use constant CMD_INCREMENTQ => 0x15; use constant CMD_DECREMENTQ => 0x16; use constant CMD_QUITQ => 0x17; use constant CMD_FLUSHQ => 0x18; use constant CMD_APPENDQ => 0x19; use constant CMD_PREPENDQ => 0x1A; use constant CMD_SASL_LIST_MECHS => 0x20; use constant CMD_SASL_AUTH => 0x21; use constant CMD_SASL_STEP => 0x22; use constant ERR_AUTH_ERROR => 0x20; # REQ and RES formats are divided even though they currently share # the same format, since they _could_ differ in the future. use constant REQ_PKT_FMT => "CCnCCnNNNN"; use constant RES_PKT_FMT => "CCnCCnNNNN"; use constant INCRDECR_PKT_FMT => "NNNNN"; use constant MIN_RECV_BYTES => length(pack(RES_PKT_FMT)); use constant REQ_MAGIC => 0x80; use constant RES_MAGIC => 0x81; my $pwd=getcwd; $ENV{'SASL_CONF_PATH'} = "$pwd/t/sasl"; my $server = new_memcached('-B binary -U 0 -S -l 127.0.0.1 '); my $mc = MC::Client->new; my $check = sub { my ($key, $orig_val) = @_; my ($status, $val, $cas) = $mc->get($key); if ($val =~ /^\d+$/) { cmp_ok($val,'==', $orig_val, "$val = $orig_val"); } else { cmp_ok($val, 'eq', $orig_val, "$val = $orig_val"); } }; my $set = sub { my ($key, $orig_value, $exp) = @_; $exp = defined $exp ? $exp : 0; my ($status, $rv)= $mc->set($key, $orig_value, $exp); $check->($key, $orig_value); }; my $empty = sub { my $key = shift; my ($status,$rv) =()= eval { $mc->get($key) }; #if ($status == ERR_AUTH_ERROR) { # ok($@->auth_error, "Not authorized to connect"); #} #else { # ok($@->not_found, "We got a not found error when we expected one"); #} if ($status) { ok($@->not_found, "We got a not found error when we expected one"); } }; my $delete = sub { my ($key, $when) = @_; $mc->delete($key, $when); $empty->($key); }; # BEGIN THE TEST ok($server, "started the server"); my $v = $mc->version; ok(defined $v && length($v), "Proper version: $v"); # list mechs my $mechs= $mc->list_mechs(); Test::More::cmp_ok($mechs, 'eq', 'CRAM-MD5 PLAIN', "list_mechs $mechs"); # this should fail, not authenticated { my ($status, $val)= $mc->set('x', "somevalue"); ok($status, "this fails to authenticate"); cmp_ok($status,'==',ERR_AUTH_ERROR, "error code matches"); } $empty->('x'); { my $mc = MC::Client->new; my ($status, $val) = $mc->delete('x'); ok($status, "this fails to authenticate"); cmp_ok($status,'==',ERR_AUTH_ERROR, "error code matches"); } $empty->('x'); { my $mc = MC::Client->new; my ($status, $val)= $mc->set('x', "somevalue"); ok($status, "this fails to authenticate"); cmp_ok($status,'==',ERR_AUTH_ERROR, "error code matches"); } $empty->('x'); { my $mc = MC::Client->new; my ($status, $val)= $mc->flush('x'); ok($status, "this fails to authenticate"); cmp_ok($status,'==',ERR_AUTH_ERROR, "error code matches"); } $empty->('x'); # Build the auth DB for testing. my $sasldb = '/tmp/test-memcached.sasldb'; unlink $sasldb; my $saslpasswd_path; for my $dir (split(/:/, $ENV{PATH}), "/usr/bin", "/usr/sbin", "/usr/local/bin", "/usr/local/sbin", ) { my $exe = $dir . '/saslpasswd2'; if (-x $exe) { $saslpasswd_path = $exe; last; } } my $sasl_realm = 'memcached.realm'; system("echo testpass | $saslpasswd_path -a memcached -u $sasl_realm -c -p testuser"); $mc = MC::Client->new; # Attempt a bad auth mech. is ($mc->authenticate('testuser', 'testpass', "X" x 40), 0x4, "bad mech"); # Attempt bad authentication. is ($mc->authenticate('testuser', 'wrongpassword'), 0x20, "bad auth"); # Now try good authentication and make the tests work. is ($mc->authenticate('testuser', 'testpass'), 0, "authenticated"); # these should work { my ($status, $val)= $mc->set('x', "somevalue"); ok(! $status); } $check->('x','somevalue'); { my ($status, $val)= $mc->delete('x'); ok(! $status); } $empty->('x'); { my ($status, $val)= $mc->set('x', "somevalue"); ok(! $status); } $check->('x','somevalue'); { my ($status, $val)= $mc->flush('x'); ok(! $status); } $empty->('x'); { my $mc = MC::Client->new; # Attempt bad authentication. is ($mc->authenticate('testuser', 'wrongpassword'), 0x20, "bad auth"); # This should fail because $mc is not authenticated my ($status, $val)= $mc->set('x', "somevalue"); ok($status, "this fails to authenticate"); cmp_ok($status,'==',ERR_AUTH_ERROR, "error code matches"); } $empty->('x', 'somevalue'); { my $mc = MC::Client->new; # Attempt bad authentication. is ($mc->authenticate('testuser', 'wrongpassword'), 0x20, "bad auth"); # Mix an authenticated connection and an unauthenticated connection to # confirm c->authenticated is not shared among connections my $mc2 = MC::Client->new; is ($mc2->authenticate('testuser', 'testpass'), 0, "authenticated"); my ($status, $val)= $mc2->set('x', "somevalue"); ok(! $status); # This should fail because $mc is not authenticated ($status, $val)= $mc->set('x', "somevalue"); ok($status, "this fails to authenticate"); cmp_ok($status,'==',ERR_AUTH_ERROR, "error code matches"); } { my $mc = MC::Client->new; is ($mc->sasl_step('testuser', 'testpass'), 0x20, "sasl_step_fails_no_segfault"); } # check the SASL stats, make sure they track things correctly # note: the enabled or not is presence checked in stats.t # while authenticated, get current counter # # My initial approach was going to be to get current counts, reauthenticate # and fail, followed by a reauth successfully so I'd know what happened. # Reauthentication is currently unsupported, so it doesn't work that way at the # moment. Adding tests may break this. { my %stats = $mc->stats(''); is ($stats{'auth_cmds'}, 6, "auth commands counted"); is ($stats{'auth_errors'}, 4, "auth errors correct"); } # Along with the assertion added to the code to verify we're staying # within bounds when we do a stats detail dump (detail turned on at # the top). # my %stats = $mc->stats('detail dump'); # ###################################################################### # Test ends around here. # ###################################################################### package MC::Client; use strict; use warnings; use fields qw(socket); use IO::Socket::INET; use constant ERR_AUTH_ERROR => 0x20; sub new { my $self = shift; my ($s) = @_; $s = $server unless defined $s; my $sock = $s->sock; $self = fields::new($self); $self->{socket} = $sock; return $self; } sub authenticate { my ($self, $user, $pass, $mech)= @_; $mech ||= 'PLAIN'; my $buf = sprintf("%c%s@%s%c%s", 0, $user, $sasl_realm, 0, $pass); my ($status, $rv, undef) = $self->_do_command(::CMD_SASL_AUTH, $mech, $buf, ''); return $status; } sub sasl_step { my ($self, $user, $pass, $mech)= @_; $mech ||= 'PLAIN'; my $buf = sprintf("%c%s@%s%c%s", 0, $user, $sasl_realm, 0, $pass); my ($status, $rv, undef) = $self->_do_command(::CMD_SASL_STEP, $mech, $buf, ''); return $status; } sub list_mechs { my ($self)= @_; my ($status, $rv, undef) = $self->_do_command(::CMD_SASL_LIST_MECHS, '', '', ''); return join(" ", sort(split(/\s+/, $rv))); } sub build_command { my $self = shift; die "Not enough args to send_command" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; $extra_header = '' unless defined $extra_header; my $keylen = length($key); my $vallen = length($val); my $extralen = length($extra_header); my $datatype = 0; # field for future use my $reserved = 0; # field for future use my $totallen = $keylen + $vallen + $extralen; my $ident_hi = 0; my $ident_lo = 0; if ($cas) { $ident_hi = int($cas / 2 ** 32); $ident_lo = int($cas % 2 ** 32); } my $msg = pack(::REQ_PKT_FMT, ::REQ_MAGIC, $cmd, $keylen, $extralen, $datatype, $reserved, $totallen, $opaque, $ident_hi, $ident_lo); my $full_msg = $msg . $extra_header . $key . $val; return $full_msg; } sub send_command { my $self = shift; die "Not enough args to send_command" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; my $full_msg = $self->build_command($cmd, $key, $val, $opaque, $extra_header, $cas); my $sent = $self->{socket}->send($full_msg); die("Send failed: $!") unless $sent; if($sent != length($full_msg)) { die("only sent $sent of " . length($full_msg) . " bytes"); } } sub flush_socket { my $self = shift; $self->{socket}->flush; } # Send a silent command and ensure it doesn't respond. sub send_silent { my $self = shift; die "Not enough args to send_silent" unless @_ >= 4; my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_; $self->send_command($cmd, $key, $val, $opaque, $extra_header, $cas); $self->send_command(::CMD_NOOP, '', '', $opaque + 1); my ($ropaque, $status, $data) = $self->_handle_single_response; Test::More::is($ropaque, $opaque + 1); } sub silent_mutation { my $self = shift; my ($cmd, $key, $value) = @_; $empty->($key); my $extra = pack "NN", 82, 0; $mc->send_silent($cmd, $key, $value, 7278552, $extra, 0); $check->($key, $value); } sub _handle_single_response { my $self = shift; my $myopaque = shift; $self->{socket}->recv(my $response, ::MIN_RECV_BYTES); my ($magic, $cmd, $keylen, $extralen, $datatype, $status, $remaining, $opaque, $ident_hi, $ident_lo) = unpack(::RES_PKT_FMT, $response); return ($opaque, '', '', '', 0) if not defined $remaining; return ($opaque, '', '', '', 0) if ($remaining == 0); # fetch the value my $rv=""; while($remaining - length($rv) > 0) { $self->{socket}->recv(my $buf, $remaining - length($rv)); $rv .= $buf; } if(length($rv) != $remaining) { my $found = length($rv); die("Expected $remaining bytes, got $found"); } my $cas = ($ident_hi * 2 ** 32) + $ident_lo; #if ($status) { #die MC::Error->new($status, $rv); #} return ($opaque, $status, $rv, $cas, $keylen); } sub _do_command { my $self = shift; die unless @_ >= 3; my ($cmd, $key, $val, $extra_header, $cas) = @_; $extra_header = '' unless defined $extra_header; my $opaque = int(rand(2**32)); $self->send_command($cmd, $key, $val, $opaque, $extra_header, $cas); my (undef, $status, $rv, $rcas) = $self->_handle_single_response($opaque); return ($status, $rv, $rcas); } sub _incrdecr_header { my $self = shift; my ($amt, $init, $exp) = @_; my $amt_hi = int($amt / 2 ** 32); my $amt_lo = int($amt % 2 ** 32); my $init_hi = int($init / 2 ** 32); my $init_lo = int($init % 2 ** 32); my $extra_header = pack(::INCRDECR_PKT_FMT, $amt_hi, $amt_lo, $init_hi, $init_lo, $exp); return $extra_header; } sub _incrdecr { my $self = shift; my ($cmd, $key, $amt, $init, $exp) = @_; my ($status, $data, undef) = $self->_do_command($cmd, $key, '', $self->_incrdecr_header($amt, $init, $exp)); my $header = substr $data, 0, 8, ''; my ($resp_hi, $resp_lo) = unpack "NN", $header; my $resp = ($resp_hi * 2 ** 32) + $resp_lo; return $resp; } sub silent_incrdecr { my $self = shift; my ($cmd, $key, $amt, $init, $exp) = @_; my $opaque = 8275753; $mc->send_silent($cmd, $key, '', $opaque, $mc->_incrdecr_header($amt, $init, $exp)); } sub stats { my $self = shift; my $key = shift; my $cas = 0; my $opaque = int(rand(2**32)); $self->send_command(::CMD_STAT, $key, '', $opaque, '', $cas); my %rv = (); my $found_key = ''; my $found_val = ''; my $status= 0; do { my ($op, $status, $data, $cas, $keylen) = $self->_handle_single_response($opaque); if ($keylen > 0) { $found_key = substr($data, 0, $keylen); $found_val = substr($data, $keylen); $rv{$found_key} = $found_val; } else { $found_key = ''; } } while($found_key ne ''); return %rv; } sub get { my $self = shift; my $key = shift; my ($status, $rv, $cas) = $self->_do_command(::CMD_GET, $key, '', ''); my $header = substr $rv, 0, 4, ''; my $flags = unpack("N", $header); return ($status, $rv); } sub get_multi { my $self = shift; my @keys = @_; for (my $i = 0; $i < @keys; $i++) { $self->send_command(::CMD_GETQ, $keys[$i], '', $i, '', 0); } my $terminal = @keys + 10; $self->send_command(::CMD_NOOP, '', '', $terminal); my %return; my $status = 0; while (1) { my ($opaque, $status, $data) = $self->_handle_single_response; last if $opaque == $terminal; my $header = substr $data, 0, 4, ''; my $flags = unpack("N", $header); $return{$keys[$opaque]} = [$flags, $data]; } return %return if wantarray; return \%return; } sub version { my $self = shift; return $self->_do_command(::CMD_VERSION, '', ''); } sub flush { my $self = shift; return $self->_do_command(::CMD_FLUSH, '', ''); } sub add { my $self = shift; my ($key, $val, $flags, $expire) = @_; my $extra_header = pack "NN", $flags, $expire; my $cas = 0; return $self->_do_command(::CMD_ADD, $key, $val, $extra_header, $cas); } sub set { my $self = shift; my $flags = 0; my $cas = 0; my ($key, $val, $expire) = @_; $expire = defined $expire ? $expire : 0; my $extra_header = pack "NN", $flags, $expire; return $self->_do_command(::CMD_SET, $key, $val, $extra_header, $cas); } sub _append_prepend { my $self = shift; my ($cmd, $key, $val, $cas) = @_; return $self->_do_command($cmd, $key, $val, '', $cas); } sub replace { my $self = shift; my ($key, $val, $flags, $expire) = @_; my $extra_header = pack "NN", $flags, $expire; my $cas = 0; return $self->_do_command(::CMD_REPLACE, $key, $val, $extra_header, $cas); } sub delete { my $self = shift; my ($key) = @_; return $self->_do_command(::CMD_DELETE, $key, ''); } sub incr { my $self = shift; my ($key, $amt, $init, $exp) = @_; $amt = 1 unless defined $amt; $init = 0 unless defined $init; $exp = 0 unless defined $exp; return $self->_incrdecr(::CMD_INCR, $key, $amt, $init, $exp); } sub decr { my $self = shift; my ($key, $amt, $init, $exp) = @_; $amt = 1 unless defined $amt; $init = 0 unless defined $init; $exp = 0 unless defined $exp; return $self->_incrdecr(::CMD_DECR, $key, $amt, $init, $exp); } sub noop { my $self = shift; return $self->_do_command(::CMD_NOOP, '', ''); } package MC::Error; use strict; use warnings; use constant ERR_UNKNOWN_CMD => 0x81; use constant ERR_NOT_FOUND => 0x1; use constant ERR_EXISTS => 0x2; use constant ERR_TOO_BIG => 0x3; use constant ERR_EINVAL => 0x4; use constant ERR_NOT_STORED => 0x5; use constant ERR_DELTA_BADVAL => 0x6; use constant ERR_AUTH_ERROR => 0x20; use overload '""' => sub { my $self = shift; return "Memcache Error ($self->[0]): $self->[1]"; }; sub new { my $class = shift; my $error = [@_]; my $self = bless $error, (ref $class || $class); return $self; } sub not_found { my $self = shift; return $self->[0] == ERR_NOT_FOUND; } sub exists { my $self = shift; return $self->[0] == ERR_EXISTS; } sub too_big { my $self = shift; return $self->[0] == ERR_TOO_BIG; } sub delta_badval { my $self = shift; return $self->[0] == ERR_DELTA_BADVAL; } sub auth_error { my $self = shift; return $self->[0] == ERR_AUTH_ERROR; } unlink $sasldb; # vim: filetype=perl memcached-1.5.22/t/chunked-extstore.t0000664000175000017500000001205013550051203014346 00000000000000#!/usr/bin/perl # Networked logging tests. use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $ext_path; if (!supports_extstore()) { plan skip_all => 'extstore not enabled'; exit 0; } $ext_path = "/tmp/extstore.$$"; my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,slab_chunk_max=16384,slab_automove=0,ext_compact_under=1"); my $sock = $server->sock; # Wait until all items have flushed sub wait_for_ext { my $sum = 1; while ($sum != 0) { my $s = mem_stats($sock, "items"); $sum = 0; for my $key (keys %$s) { if ($key =~ m/items:(\d+):number/) { # Ignore classes which can contain extstore items next if $1 < 3; $sum += $s->{$key}; } } sleep 1 if $sum != 0; } } # We're testing to ensure item chaining doesn't corrupt or poorly overlap # data, so create a non-repeating pattern. my @parts = (); for (1 .. 8000) { push(@parts, $_); } my $pattern = join(':', @parts); my $plen = length($pattern); print $sock "set pattern 0 0 $plen\r\n$pattern\r\n"; is(scalar <$sock>, "STORED\r\n", "stored pattern successfully"); # Stash two more for later test print $sock "set pattern2 0 0 $plen noreply\r\n$pattern\r\n"; print $sock "set pattern3 0 0 $plen noreply\r\n$pattern\r\n"; sleep 4; mem_get_is($sock, "pattern", $pattern); for (1..5) { my $size = 400 * 1024; my $data = "x" x $size; print $sock "set foo$_ 0 0 $size\r\n$data\r\n"; my $res = <$sock>; is($res, "STORED\r\n", "stored some big items"); } { my $max = 1024 * 1024; my $big = "a big value that's > .5M and < 1M. "; while (length($big) * 2 < $max) { $big = $big . $big; } my $biglen = length($big); for (1..40) { print $sock "set toast$_ 0 0 $biglen\r\n$big\r\n"; is(scalar <$sock>, "STORED\r\n", "stored big"); } wait_for_ext(); for (1..40) { mem_get_is($sock, "toast$_", $big); } my $stats = mem_stats($sock); cmp_ok($stats->{extstore_page_allocs}, '>', 0, 'at least one page allocated'); cmp_ok($stats->{extstore_objects_written}, '>', 25, 'some objects written'); cmp_ok($stats->{extstore_bytes_written}, '>', $max * 2, 'some bytes written'); cmp_ok($stats->{get_extstore}, '>', 5, 'multiple objects fetched'); cmp_ok($stats->{extstore_objects_read}, '>', 0, 'one object read'); cmp_ok($stats->{extstore_bytes_read}, '>', $max * 2, 'some bytes read'); is($stats->{badcrc_from_extstore}, 0, 'CRC checks successful'); } # fill to eviction { my $keycount = 1250; for (1 .. $keycount) { print $sock "set mfoo$_ 0 0 $plen noreply\r\n$pattern\r\n"; wait_for_ext() if $_ % 500 == 0; } # because item_age is set to 2s. wait_for_ext(); my $stats = mem_stats($sock); cmp_ok($stats->{extstore_page_evictions}, '>', 0, 'at least one page evicted'); cmp_ok($stats->{extstore_objects_evicted}, '>', 0, 'at least one object evicted'); cmp_ok($stats->{extstore_bytes_evicted}, '>', 0, 'some bytes evicted'); cmp_ok($stats->{extstore_pages_free}, '<', 2, 'most pages are used'); is($stats->{miss_from_extstore}, 0, 'no misses'); # original "pattern" key should be gone. mem_get_is($sock, "pattern", undef, "original pattern key is gone"); $stats = mem_stats($sock); is($stats->{miss_from_extstore}, 1, 'one extstore miss'); print $sock "get pattern2 pattern3\r\n"; is(scalar <$sock>, "END\r\n", "multiget double miss"); $stats = mem_stats($sock); is($stats->{miss_from_extstore}, 3, 'three extstore misses'); } # Let compaction run. { for (1..40) { print $sock "delete toast$_ noreply\r\n" if $_ % 2 == 0; } for (1..1250) { # Force a read so objects don't get skipped. print $sock "add mfoo$_ 0 0 1 noreply\r\n1\r\n" if $_ % 2 == 1; } for (1..1250) { # Delete lots of objects to trigger compaction. print $sock "delete mfoo$_ noreply\r\n" if $_ % 2 == 0; } print $sock "extstore compact_under 4\r\n"; my $res = <$sock>; print $sock "extstore drop_under 3\r\n"; $res = <$sock>; sleep 4; my $stats = mem_stats($sock); cmp_ok($stats->{extstore_pages_free}, '>', 2, 'some pages now free'); cmp_ok($stats->{extstore_compact_rescues}, '>', 0, 'some compaction rescues happened'); # Some of the early items got evicted for (750..1250) { # everything should validate properly. mem_get_is($sock, "mfoo$_", $pattern) if $_ % 2 == 1; } } # test recache { print $sock "extstore recache_rate 1\r\n"; is(scalar <$sock>, "OK\r\n", "upped recache rate"); for (1150..1250) { mem_get_is($sock, "mfoo$_", $pattern) if $_ % 2 == 1; } my $stats = mem_stats($sock); cmp_ok($stats->{recache_from_extstore}, '>', 25, 'recaching happening'); } done_testing(); END { unlink $ext_path if $ext_path; } memcached-1.5.22/t/stress-memcached.pl0000755000175000017500000000426511246331452014472 00000000000000#!/usr/bin/perl # use strict; use lib '../../api/perl/lib'; use Cache::Memcached; use Time::HiRes qw(time); unless (@ARGV == 2) { die "Usage: stress-memcached.pl ip:port threads\n"; } my $host = shift; my $threads = shift; my $memc = new Cache::Memcached; $memc->set_servers([$host]); unless ($memc->set("foo", "bar") && $memc->get("foo") eq "bar") { die "memcached not running at $host ?\n"; } $memc->disconnect_all(); my $running = 0; while (1) { if ($running < $threads) { my $cpid = fork(); if ($cpid) { $running++; #print "Launched $cpid. Running $running threads.\n"; } else { stress(); exit 0; } } else { wait(); $running--; } } sub stress { undef $memc; $memc = new Cache::Memcached; $memc->set_servers([$host]); my ($t1, $t2); my $start = sub { $t1 = time(); }; my $stop = sub { my $op = shift; $t2 = time(); my $td = sprintf("%0.3f", $t2 - $t1); if ($td > 0.25) { print "Took $td seconds for: $op\n"; } }; my $max = rand(50); my $sets = 0; for (my $i = 0; $i < $max; $i++) { my $key = key($i); my $set = $memc->set($key, $key); $sets++ if $set; } for (1..int(rand(500))) { my $rand = int(rand($max)); my $key = key($rand); my $meth = int(rand(3)); my $exp = int(rand(3)); undef $exp unless $exp; $start->(); if ($meth == 0) { $memc->add($key, $key, $exp); $stop->("add"); } elsif ($meth == 1) { $memc->delete($key); $stop->("delete"); } else { $memc->set($key, $key, $exp); $stop->("set"); } $rand = int(rand($max)); $key = key($rand); $start->(); my $v = $memc->get($key); $stop->("get"); if ($v && $v ne $key) { die "Bogus: $v for key $rand\n"; } } $start->(); my $multi = $memc->get_multi(map { key(int(rand($max))) } (1..$max)); $stop->("get_multi"); } sub key { my $n = shift; $_ = sprintf("%04d", $n); if ($n % 2) { $_ .= "a"x20; } $_; } memcached-1.5.22/t/extstore-jbod.t0000664000175000017500000000400613461170555013662 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use Data::Dumper qw/Dumper/; my $ext_path; my $ext_path2; if (!supports_extstore()) { plan skip_all => 'extstore not enabled'; exit 0; } $ext_path = "/tmp/extstore1.$$"; $ext_path2 = "/tmp/extstore2.$$"; my $server = new_memcached("-m 256 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,ext_path=$ext_path2:96m,slab_automove=1"); my $sock = $server->sock; my $value; { my @chars = ("C".."Z"); for (1 .. 20000) { $value .= $chars[rand @chars]; } } # fill some larger objects { # interleave sets with 0 ttl vs long ttl's. my $keycount = 3700; for (1 .. $keycount) { print $sock "set nfoo$_ 0 0 20000 noreply\r\n$value\r\n"; print $sock "set lfoo$_ 0 0 20000 noreply\r\n$value\r\n"; } # wait for a flush wait_ext_flush($sock); # delete half mem_get_is($sock, "nfoo1", $value); for (1 .. $keycount) { print $sock "delete lfoo$_ noreply\r\n"; } print $sock "lru_crawler crawl all\r\n"; <$sock>; sleep 10; # fetch # check extstore counters my $stats = mem_stats($sock); is($stats->{evictions}, 0, 'no RAM evictions'); cmp_ok($stats->{extstore_page_allocs}, '>', 0, 'at least one page allocated'); cmp_ok($stats->{extstore_objects_written}, '>', $keycount / 2, 'some objects written'); cmp_ok($stats->{extstore_bytes_written}, '>', length($value) * 2, 'some bytes written'); cmp_ok($stats->{get_extstore}, '>', 0, 'one object was fetched'); cmp_ok($stats->{extstore_objects_read}, '>', 0, 'one object read'); cmp_ok($stats->{extstore_bytes_read}, '>', length($value), 'some bytes read'); cmp_ok($stats->{extstore_page_reclaims}, '>', 1, 'at least two pages reclaimed'); } done_testing(); END { unlink $ext_path if $ext_path; unlink $ext_path2 if $ext_path2; } memcached-1.5.22/t/issue_42.t0000664000175000017500000000140113575022500012512 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 11; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached("-o no_modern"); my $sock = $server->sock; my $value = "B"x10; my $key = 0; my $key_count = 10; for ($key = 0; $key < $key_count; $key++) { print $sock "set key$key 0 0 10\r\n$value\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key$key"); } my $stats = mem_stats($sock, "slabs"); my $stats2 = mem_stats($sock, "items"); my $req = $stats2->{"items:1:mem_requested"}; my $top = $stats->{"1:chunk_size"} * $key_count; # unreasonable for the result to be < 500 bytes (min item header is 48), but # should be less than the maximum potential number. ok ($req > 500 && $req < $top, "Check allocated size: $req $top"); memcached-1.5.22/t/stats-conns.t0000775000175000017500000000422713575022500013345 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 12; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; ## First make sure we report UNIX-domain sockets correctly my $filename = "/tmp/memcachetest$$"; my $server = new_memcached("-s $filename"); my $sock = $server->sock; my $stats_sock = $server->new_sock; ok(-S $filename, "creating unix domain socket $filename"); print $sock "set foo 0 0 6\r\n"; sleep(1); # so we can test secs_since_last_cmd is nonzero print $stats_sock "stats conns\r\n"; my $stats; while (<$stats_sock>) { last if /^(\.|END)/; $stats .= $_; } like($stats, qr/STAT \d+:addr /); $stats =~ m/STAT (\d+):addr unix:(.*[^\r\n])/g; my $listen_fd = $1; my $socket_path = $2; # getsockname(2) doesn't return socket path on GNU/Hurd (and maybe others) SKIP: { skip "socket path checking on GNU kernel", 1 if ($^O eq 'gnu'); is($socket_path, $filename, "unix domain socket path reported correctly"); }; $stats =~ m/STAT (\d+):state conn_listening\r\n/g; is($1, $listen_fd, "listen socket fd reported correctly"); like($stats, qr/STAT \d+:state conn_nread/, "one client is sending data"); like($stats, qr/STAT \d+:state conn_parse_cmd/, "one client is in command processing"); like($stats, qr/STAT \d+:secs_since_last_cmd [1-9]\r/, "nonzero secs_since_last_cmd"); like($stats, qr/STAT \d+:listen_addr unix:\/tmp\/memcachetest\d+\r/, "found listen_addr for the UNIX-domain socket"); $server->stop; unlink($filename); ## Now look at TCP $server = new_memcached("-l 0.0.0.0"); $sock = $server->sock; $stats_sock = $server->new_sock; print $sock "set foo 0 0 6\r\n"; print $stats_sock "stats conns\r\n"; $stats = ''; while (<$stats_sock>) { last if /^(\.|END)/; $stats .= $_; } like($stats, qr/STAT \d+:state conn_listen/, "there is a listen socket"); $stats =~ m/STAT \d+:addr udp:0.0.0.0:(\d+)/; is($1, $server->udpport, "udp port number is correct"); $stats =~ m/STAT \d+:addr tcp:0.0.0.0:(\d+)/; print STDERR "PORT: ", $server->port, "\n"; is($1, $server->port, "tcp port number is correct"); $stats =~ m/STAT \d+:listen_addr tcp:0.0.0.0:(\d+)/; is($1, $server->port, "listen_addr is correct for the tcp port"); memcached-1.5.22/t/malicious-commands.t0000664000175000017500000000146413461170555014662 00000000000000#!/usr/bin/perl # These command strings are always expected to be malicious and as such we # should just hang up on them. use strict; use Test::More tests => 3; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my @strs = ( "GET / HTTP/1.0", "PUT /asdf/asd/fasdfasdf/sadf HTTP/1.1", "DELETE HTTP/1.1" ); for my $str (@strs) { my $server = new_memcached(); my $sock = $server->sock; print $sock "$str\r\n"; # Five seconds ought to be enough to get hung up on. my $oldalarmt = alarm(5); # Verify we can't read anything. my $bytesread = -1; eval { local $SIG{'ALRM'} = sub { die "timeout" }; my $data = ""; $bytesread = sysread($sock, $data, 24), }; is($bytesread, 0, $str); # Restore signal stuff. alarm($oldalarmt); } memcached-1.5.22/t/issue_22.t0000664000175000017500000000216613250303500012511 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 84; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached("-m 3"); my $sock = $server->sock; my $value = "B"x66560; my $key = 0; for ($key = 0; $key < 40; $key++) { print $sock "set key$key 0 0 66560\r\n$value\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key$key"); } my $first_stats = mem_stats($sock, "items"); my $first_evicted = $first_stats->{"items:31:evicted"}; # I get 1 eviction on a 32 bit binary, but 4 on a 64 binary.. # Just check that I have evictions... isnt ($first_evicted, "0", "check evicted"); print $sock "stats reset\r\n"; is (scalar <$sock>, "RESET\r\n", "Stats reset"); my $second_stats = mem_stats($sock, "items"); my $second_evicted = $second_stats->{"items:31:evicted"}; is ($second_evicted, "0", "check evicted"); for ($key = 40; $key < 80; $key++) { print $sock "set key$key 0 0 66560\r\n$value\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key$key"); } my $last_stats = mem_stats($sock, "items"); my $last_evicted = $last_stats->{"items:31:evicted"}; is ($last_evicted, "40", "check evicted"); memcached-1.5.22/t/metaget.t0000664000175000017500000004143113615376577012537 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; # command syntax: # mg [key] [flags] [tokens]\r\n # response: # VA [flags] [tokens]\r\n # data\r\n # EN\r\n # # flags: # - s: item size # - v: return item value # - c: return item cas # - t: return item TTL remaining (-1 for unlimited) # - f: client flags # - l: last access time # - h: whether item has been hit before # - O: opaque to copy back. # - k: return key # - q: noreply semantics. # - u: don't bump the item # updaters: # - N (token): vivify on miss, takes TTL as a argument # - R (token): if token is less than item TTL win for recache # - T (token): update remaining TTL # FIXME: do I need a "if stale and no token sent, flip" explicit flag? # extra response flags: # - W: client has "won" the token # - X: object is stale # - Z: object has sent a winning token # # ms [key] [flags] [tokens]\r\n # value\r\n # response: # ST [flags] [tokens]\r\n # ST STORED, NS NOT_STORED, EX EXISTS, NF NOT_FOUND # # flags: # - q: noreply # - F (token): set client flags # - C (token): compare CAS value # - S (token): item size # - T (token): TTL # - O: opaque to copy back. # - k: return key # - I: invalid. set-to-invalid if CAS is older than it should be. # Not implemented: # - E: add if not exists (influences other options) # - A: append (exclusive) # - P: prepend (exclusive) # - L: replace (exclusive) # - incr/decr? pushing it, I guess. # # md [key] [flags] [tokens]\r\n # response: # DE [flags] [tokens] # flags: # - q: noreply # - T (token): updates TTL # - C (token): compare CAS value # - I: invalidate. mark as stale, bumps CAS. # - O: opaque to copy back. # - k: return key # # mn\r\n # response: # EN # metaget tests # basic test # - raw mget # - raw mget miss # - raw mget bad key { print $sock "set foo 0 0 2\r\nhi\r\n"; is(scalar <$sock>, "STORED\r\n", "stored test value"); print $sock "me none\r\n"; is(scalar <$sock>, "EN\r\n", "raw mget miss"); print $sock "me foo\r\n"; like(scalar <$sock>, qr/^ME foo /, "raw mget result"); # bleed the EN off the socket. my $dud = scalar <$sock>; } # mget with arguments # - set some specific TTL and get it back (within reason) # - get cas # - autovivify and bit-win { print $sock "set foo2 0 90 2\r\nho\r\n"; is(scalar <$sock>, "STORED\r\n", "stored test value"); mget_is({ sock => $sock, flags => 'sv', etokens => [2] }, 'foo2', 'ho', "retrieved test value"); my $res = mget($sock, 'foo2', 'stv'); } # lease-test, use two sockets? one socket should be fine, actually. # - get a win on autovivify # - get a loss on the same command # - have a set/cas fail # - have a cas succeed # - repeat for "triggered on TTL" # - test just modifying the TTL (touch) # - test fetching without value { my $res = mget($sock, 'needwin', 'scvNt 30'); like($res->{flags}, qr/scvNt/, "got main flags back"); like($res->{flags}, qr/W/, "got a win result"); unlike($res->{flags}, qr/Z/, "no token already sent warning"); # asked for size and TTL. size should be 0, TTL should be > 0 and < 30 is($res->{tokens}->[0], 0, "got zero size: autovivified response"); my $ttl = $res->{tokens}->[1]; ok($ttl > 0 && $ttl <= 30, "auto TTL is within requested window"); # try to fail this time. { my $res = mget($sock, 'needwin', 'stcvN 30'); ok(keys %$res, "got a non-empty response"); unlike($res->{flags}, qr/W/, "not a win result"); like($res->{flags}, qr/Z/, "object already sent win result"); } # set back with the wrong CAS print $sock "ms needwin CST 5000 2 120\r\nnu\r\n"; like(scalar <$sock>, qr/^EX /, "failed to SET: CAS didn't match"); # again, but succeed. # TODO: the actual CAS command should work here too? my $cas = $res->{tokens}->[1]; print $sock "ms needwin CST $cas 2 120\r\nmu\r\n"; like(scalar <$sock>, qr/^ST /, "SET: CAS matched"); # now we repeat the original mget, but the data should be different. $res = mget($sock, 'needwin', 'sktcvN 30'); ok(keys %$res, "not a miss"); like($res->{flags}, qr/sktcvN/, "got main flags back"); unlike($res->{flags}, qr/[WZ]/, "not a win or token result"); is($res->{tokens}->[1], 'needwin', "key matches"); $ttl = $res->{tokens}->[2]; ok($ttl > 100 && $ttl <= 120, "TTL is within requested window: $ttl"); is($res->{val}, "mu", "value matches"); # now we do the whole routine again, but for "triggered on TTL being low" # TTL was set to 120 just now, so anything lower than this should trigger. $res = mget($sock, 'needwin', 'stcvNR 30 130'); like($res->{flags}, qr/stcvNR/, "got main flags back"); like($res->{flags}, qr/W/, "got a win result"); unlike($res->{flags}, qr/Z/, "no token already sent warning"); is($res->{val}, "mu", "value matches"); # try to fail this time. { my $res = mget($sock, 'needwin', 'stcvNR 30 130'); ok(keys %$res, "got a non-empty response"); unlike($res->{flags}, qr/W/, "not a win result"); like($res->{flags}, qr/Z/, "object already sent win result"); is($res->{val}, "mu", "value matches"); } # again, but succeed. $cas = $res->{tokens}->[2]; print $sock "ms needwin CST $cas 4 300\r\nzuuu\r\n"; like(scalar <$sock>, qr/^ST /, "SET: CAS matched"); # now we repeat the original mget, but the data should be different. $res = mget($sock, 'needwin', 'stcvN 30'); ok(keys %$res, "not a miss"); like($res->{flags}, qr/stcvN/, "got main flags back"); unlike($res->{flags}, qr/[WZ]/, "not a win or token result"); $ttl = $res->{tokens}->[1]; ok($ttl > 250 && $ttl <= 300, "TTL is within requested window"); ok($res->{tokens}->[0] == 4, "Size returned correctly"); is($res->{val}, "zuuu", "value matches: " . $res->{val}); } # test get-and-touch mode { # Set key with lower initial TTL. print $sock "ms gatkey ST 4 100\r\nooom\r\n"; like(scalar <$sock>, qr/^ST /, "set gatkey"); # Coolish side feature and/or bringer of bugs: 't' before 'T' gives TTL # before adjustment. 'T' before 't' gives TTL after adjustment. # Here we want 'T' before 't' to ensure we did adjust the value. my $res = mget($sock, 'gatkey', 'svTt 300'); ok(keys %$res, "not a miss"); unlike($res->{flags}, qr/[WZ]/, "not a win or token result"); my $ttl = $res->{tokens}->[1]; ok($ttl > 280 && $ttl <= 300, "TTL is within requested window: $ttl"); } # test no-value mode { # Set key with lower initial TTL. print $sock "ms hidevalue ST 4 100\r\nhide\r\n"; like(scalar <$sock>, qr/^ST /, "set hidevalue"); my $res = mget($sock, 'hidevalue', 'st'); ok(keys %$res, "not a miss"); is($res->{val}, '', "no value returned"); $res = mget($sock, 'hidevalue', 'stv'); ok(keys %$res, "not a miss"); is($res->{val}, 'hide', "real value returned"); } # test hit-before? flag { print $sock "ms hitflag ST 3 100\r\nhit\r\n"; like(scalar <$sock>, qr/^ST /, "set hitflag"); my $res = mget($sock, 'hitflag', 'sth'); ok(keys %$res, "not a miss"); is($res->{tokens}->[2], 0, "not been hit before"); $res = mget($sock, 'hitflag', 'sth'); ok(keys %$res, "not a miss"); is($res->{tokens}->[2], 1, "been hit before"); } # test no-update flag { print $sock "ms noupdate ST 3 100\r\nhit\r\n"; like(scalar <$sock>, qr/^ST /, "set noupdate"); my $res = mget($sock, 'noupdate', 'stuh'); ok(keys %$res, "not a miss"); is($res->{tokens}->[2], 0, "not been hit before"); # _next_ request should show a hit. # gets modified here but returns previous state. $res = mget($sock, 'noupdate', 'sth'); is($res->{tokens}->[2], 0, "still not a hit"); $res = mget($sock, 'noupdate', 'stuh'); is($res->{tokens}->[2], 1, "finally a hit"); } # test last-access time { print $sock "ms la_test ST 2 100\r\nla\r\n"; like(scalar <$sock>, qr/^ST /, "set la_test"); sleep 2; my $res = mget($sock, 'la_test', 'stl'); ok(keys %$res, "not a miss"); print STDERR "Last access is: ", $res->{tokens}->[2], "\n"; isnt($res->{tokens}->[2], 0, "been over a second since most recently accessed"); # TODO: Can't test re-accessing since it requires a long wait right now. # I want to adjust the LA time accuracy in a deliberate change. } # high level tests: # - mget + mset with serve-stale # - set a value # - mget it back. should be no XZW tokens # - invalidate via mdelete and mget/revalidate with mset # - remember failure scenarios! # - TTL timed out? # - CAS too high? # - also test re-setting as stale (CAS is below requested) # - this should probably be conditional. { diag "starting serve stale with mdelete"; my ($ttl, $cas, $res); print $sock "set toinv 0 0 3\r\nmoo\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key 'toinv'"); $res = mget($sock, 'toinv', 'sv'); unlike($res->{flags}, qr/[XWZ]/, "no extra flags"); # Lets mark the sucker as invalid, and drop its TTL to 30s diag "running mdelete"; print $sock "md toinv IT 30\r\n"; like(scalar <$sock>, qr/^DE /, "mdelete'd key"); # TODO: decide on if we need an explicit flag for "if I fetched a stale # value, does winning matter? # I think it's probably fine. clients can always ignore the win, or we can # add an option later to "don't try to revalidate if stale", perhaps. $res = mget($sock, 'toinv', 'stcv'); ok(keys %$res, "not a miss"); like($res->{flags}, qr/stcv/, "got main flags back"); like($res->{flags}, qr/W/, "won the recache"); like($res->{flags}, qr/X/, "item is marked stale"); $ttl = $res->{tokens}->[1]; ok($ttl > 0 && $ttl <= 30, "TTL is within requested window"); ok($res->{tokens}->[0] == 3, "Size returned correctly"); is($res->{val}, "moo", "value matches"); diag "trying to fail then stale set via mset"; print $sock "ms toinv STC 1 90 0\r\nf\r\n"; like(scalar <$sock>, qr/^EX /, "failed to SET: low CAS didn't match"); print $sock "ms toinv SITC 1 90 0\r\nf\r\n"; like(scalar <$sock>, qr/^ST /, "SET an invalid/stale item"); diag "confirm item still stale, and TTL wasn't raised."; $res = mget($sock, 'toinv', 'stcv'); like($res->{flags}, qr/X/, "item is marked stale"); like($res->{flags}, qr/Z/, "win token already sent"); unlike($res->{flags}, qr/W/, "didn't win: token already sent"); $ttl = $res->{tokens}->[1]; ok($ttl > 0 && $ttl <= 30, "TTL wasn't modified"); # TODO: CAS too high? diag "do valid mset"; $cas = $res->{tokens}->[2]; print $sock "ms toinv STC 1 90 $cas\r\ng\r\n"; like(scalar <$sock>, qr/^ST /, "SET over the stale item"); $res = mget($sock, 'toinv', 'stcv'); ok(keys %$res, "not a miss"); unlike($res->{flags}, qr/[WXZ]/, "no stale, win, or tokens"); $ttl = $res->{tokens}->[1]; ok($ttl > 30 && $ttl <= 90, "TTL was modified"); ok($cas != $res->{tokens}->[2], "CAS was updated"); is($res->{tokens}->[0], 1, "size updated"); is($res->{val}, "g", "value was updated"); } # Quiet flag suppresses most output. Badly invalid commands will still # generate something. Not weird to parse like 'noreply' token was... # mget's with hits should return real data. { diag "testing quiet flag"; print $sock "ms quiet Sq 2\r\nmo\r\n"; print $sock "md quiet q\r\n"; print $sock "mg quiet svq\r\n"; diag "now purposefully cause an error\r\n"; print $sock "ms quiet S\r\n"; like(scalar <$sock>, qr/^CLIENT_ERROR/, "resp not ST, DE, or EN"); # Now try a pipelined get. Throw an mnop at the end print $sock "ms quiet Sq 2\r\nbo\r\n"; print $sock "mg quiet svq\r\nmg quiet svq\r\nmg quietmiss svq\r\nmn\r\n"; # Should get back VA/data/VA/data/EN like(scalar <$sock>, qr/^VA svq 2/, "get response"); like(scalar <$sock>, qr/^bo/, "get value"); like(scalar <$sock>, qr/^VA svq 2/, "get response"); like(scalar <$sock>, qr/^bo/, "get value"); like(scalar <$sock>, qr/^EN/, "end token"); } { my $k = 'otest'; diag "testing mget opaque"; print $sock "ms $k ST 2 100\r\nra\r\n"; like(scalar <$sock>, qr/^ST /, "set $k"); my $res = mget($sock, $k, 'stvO opaque'); is($res->{tokens}->[2], 'opaque', "O flag returned opaque"); } { diag "flag and token count errors"; print $sock "mg foo sv extratoken\r\n"; like(scalar <$sock>, qr/^CLIENT_ERROR incorrect number of tokens/, "too many tokens"); print $sock "mg foo svN\r\n"; like(scalar <$sock>, qr/^CLIENT_ERROR incorrect number of tokens/, "too few tokens"); print $sock "mg foo mooooo\r\n"; like(scalar <$sock>, qr/^CLIENT_ERROR invalid or duplicate flag/, "gone silly with flags"); } # TODO: move wait_for_ext into Memcached.pm sub wait_for_ext { my $sock = shift; my $target = shift || 0; my $sum = $target + 1; while ($sum > $target) { my $s = mem_stats($sock, "items"); $sum = 0; for my $key (keys %$s) { if ($key =~ m/items:(\d+):number/) { # Ignore classes which can contain extstore items next if $1 < 3; $sum += $s->{$key}; } } sleep 1 if $sum > $target; } } my $ext_path; # Do a basic extstore test if enabled. if (supports_extstore()) { diag "mget + extstore tests"; $ext_path = "/tmp/extstore.$$"; my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,slab_automove=0,ext_compact_under=1,no_lru_crawler"); my $sock = $server->sock; my $value; { my @chars = ("C".."Z"); for (1 .. 20000) { $value .= $chars[rand @chars]; } } my $keycount = 10; for (1 .. $keycount) { print $sock "set nfoo$_ 0 0 20000 noreply\r\n$value\r\n"; } wait_for_ext($sock); mget_is({ sock => $sock, flags => 'sv', etokens => [20000] }, 'nfoo1', $value, "retrieved test value"); my $stats = mem_stats($sock); cmp_ok($stats->{get_extstore}, '>', 0, 'one object was fetched'); my $ovalue = $value; for (1 .. 4) { $value .= $ovalue; } # Fill to eviction. $keycount = 1000; for (1 .. $keycount) { print $sock "set mfoo$_ 0 0 100000 noreply\r\n$value\r\n"; # wait to avoid memory evictions wait_for_ext($sock, 1) if ($_ % 250 == 0); } print $sock "mg mfoo1 sv\r\n"; is(scalar <$sock>, "EN\r\n"); print $sock "mg mfoo1 svq\r\nmn\r\n"; is(scalar <$sock>, "EN\r\n"); $stats = mem_stats($sock); cmp_ok($stats->{miss_from_extstore}, '>', 0, 'at least one miss'); } ### # takes hash: # - sock # - args (metaget flags) # - array of tokens # - array of expected response tokens # returns hash: # - win (if won a condition) # - array of tokens # - value, etc? # useful to chain together for further requests. # works only with single line values. no newlines in value. # FIXME: some workaround for super long values :| # TODO: move this to lib/MemcachedTest.pm sub mget_is { # single line values only my ($o, $key, $val, $msg) = @_; my $dval = defined $val ? "'$val'" : ""; $msg ||= "$key == $dval"; my $s = $o->{sock}; my $flags = $o->{flags}; # sometimes response flags can differ from request flags. my $eflags = $o->{eflags} || $flags; my $tokens = exists $o->{tokens} ? join(' ', @{$o->{tokens}}) : ''; my $etokens = exists $o->{etokens} ? join(' ', @{$o->{etokens}}) : ''; print $s "mg $key $flags $tokens\r\n"; if (! defined $val) { my $line = scalar <$s>; if ($line =~ /^VA/) { $line .= scalar(<$s>) . scalar(<$s>); } Test::More::is($line, "EN\r\n", $msg); } else { my $len = length($val); my $body = scalar(<$s>); my $expected = "VA $eflags $etokens\r\n$val\r\nEN\r\n"; if (!$body || $body =~ /^EN/) { Test::More::is($body, $expected, $msg); return; } $body .= scalar(<$s>) . scalar(<$s>); Test::More::is($body, $expected, $msg); return mget_res($body); } return {}; } sub mget { my $s = shift; my $key = shift; my $flags = shift; my $tokens = join(' ', @_); print $s "mg $key $flags ", $tokens, "\r\n"; my $header = scalar(<$s>); my $val = "\r\n"; if ($flags =~ m/v/) { $val = scalar(<$s>); } my $end = scalar(<$s>); return mget_res($header . $val); } # parse out a response sub mget_res { my $resp = shift; my %r = (); if ($resp =~ m/^VA ([^\s]+) ([^\r]+)\r\n(.*)\r\n/gm) { $r{flags} = $1; $r{val} = $3; $r{tokens} = [split(/ /, $2)]; } return \%r; } done_testing(); END { unlink $ext_path if $ext_path; } memcached-1.5.22/t/server_crt.pem0000664000175000017500000000712513611710715013565 00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 1 (0x1) Signature Algorithm: sha1WithRSAEncryption Issuer: CN=Test Root Certificate Authority, ST=CA, C=US/emailAddress=root@test.com, O=Test, OU=Test Department Validity Not Before: Jan 7 18:25:01 2019 GMT Not After : Jan 6 18:25:01 2024 GMT Subject: CN=test.com, ST=CA, C=US/emailAddress=root@test.com, O=Test, OU=Subunit of Test Organization Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (1024 bit) Modulus: 00:ca:76:35:79:91:e8:a2:ee:ef:f4:35:7e:29:85: 75:90:65:5a:60:f8:a0:d9:ef:68:d1:61:79:69:d7: e2:7b:f1:67:71:65:50:31:4e:9a:f8:6f:27:e9:05: 0b:0e:76:95:24:9b:c2:bf:90:e5:6b:45:fd:e3:54: ac:d5:62:90:4e:37:de:8f:ae:96:f6:b3:57:eb:ad: b8:44:13:5d:a7:34:76:c1:26:49:91:67:3e:5e:52: 68:c1:1c:7f:91:c7:9e:01:e2:be:a7:a8:eb:3f:44: 6d:c7:c5:82:4b:97:d9:3f:c0:51:99:1b:20:df:12: a3:e6:bb:66:45:6d:b4:e1:07 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE X509v3 Key Usage: Digital Signature, Non Repudiation, Key Encipherment X509v3 Subject Alternative Name: DNS:test.com, DNS:alt.test.com Signature Algorithm: sha1WithRSAEncryption 61:c1:18:b4:04:79:05:0e:b9:79:2f:0e:3f:aa:f1:36:e3:90: b9:c6:99:25:53:4a:06:64:52:92:29:c5:09:41:a6:16:74:1b: 3c:4e:81:32:c2:d8:54:e0:1e:08:45:cf:f1:d0:ad:ea:11:1a: b5:cf:7c:98:8a:dd:c1:01:e4:d0:f5:8e:60:fa:7f:e7:74:2d: 91:43:81:bd:95:92:41:66:84:8b:8c:70:d7:2f:d4:2f:37:82: 8f:9a:ef:c0:7d:c5:56:56:92:7a:00:b6:30:65:37:4c:6c:7a: ba:cc:e2:dc:73:e9:f5:2c:3c:3e:31:67:ee:3d:b7:78:96:89: ba:be:4f:85:a2:a8:83:3e:53:20:f0:bf:29:50:dc:23:38:58: d8:33:f5:7b:4a:12:df:2b:34:4c:1c:f1:76:6b:86:95:74:43: 29:f7:68:f0:ca:04:08:89:ac:97:7d:05:14:a0:ca:81:56:5c: dd:c3:56:a3:53:01:0a:01:5c:55:b4:39:10:1b:be:47:19:58: a1:f2:e6:0c:08:95:b0:35:4e:6c:81:6d:b9:cf:0e:5c:70:ff: f3:b4:a7:95:69:1a:58:b7:ac:cc:2c:79:47:7e:20:17:cc:36: be:2c:10:11:31:28:63:dd:6f:8d:8d:e9:11:ea:ca:fc:10:0a: e0:ae:53:db -----BEGIN CERTIFICATE----- MIIDQjCCAiqgAwIBAgIBATANBgkqhkiG9w0BAQUFADCBizEoMCYGA1UEAwwfVGVz dCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eTELMAkGA1UECAwCQ0ExCzAJBgNV BAYTAlVTMRwwGgYJKoZIhvcNAQkBFg1yb290QHRlc3QuY29tMQ0wCwYDVQQKDARU ZXN0MRgwFgYDVQQLDA9UZXN0IERlcGFydG1lbnQwHhcNMTkwMTA3MTgyNTAxWhcN MjQwMTA2MTgyNTAxWjCBgTERMA8GA1UEAwwIdGVzdC5jb20xCzAJBgNVBAgMAkNB MQswCQYDVQQGEwJVUzEcMBoGCSqGSIb3DQEJARYNcm9vdEB0ZXN0LmNvbTENMAsG A1UECgwEVGVzdDElMCMGA1UECwwcU3VidW5pdCBvZiBUZXN0IE9yZ2FuaXphdGlv bjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAynY1eZHoou7v9DV+KYV1kGVa YPig2e9o0WF5adfie/FncWVQMU6a+G8n6QULDnaVJJvCv5Dla0X941Ss1WKQTjfe j66W9rNX6624RBNdpzR2wSZJkWc+XlJowRx/kceeAeK+p6jrP0Rtx8WCS5fZP8BR mRsg3xKj5rtmRW204QcCAwEAAaM9MDswCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAw IQYDVR0RBBowGIIIdGVzdC5jb22CDGFsdC50ZXN0LmNvbTANBgkqhkiG9w0BAQUF AAOCAQEAYcEYtAR5BQ65eS8OP6rxNuOQucaZJVNKBmRSkinFCUGmFnQbPE6BMsLY VOAeCEXP8dCt6hEatc98mIrdwQHk0PWOYPp/53QtkUOBvZWSQWaEi4xw1y/ULzeC j5rvwH3FVlaSegC2MGU3TGx6uszi3HPp9Sw8PjFn7j23eJaJur5PhaKogz5TIPC/ KVDcIzhY2DP1e0oS3ys0TBzxdmuGlXRDKfdo8MoECImsl30FFKDKgVZc3cNWo1MB CgFcVbQ5EBu+RxlYofLmDAiVsDVObIFtuc8OXHD/87SnlWkaWLeszCx5R34gF8w2 viwQETEoY91vjY3pEerK/BAK4K5T2w== -----END CERTIFICATE----- memcached-1.5.22/t/lru.t0000775000175000017500000000315213461170555011677 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 149; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; # assuming max slab is 1M and default mem is 64M my $server = new_memcached('-o no_modern'); my $sock = $server->sock; # create a big value for the largest slab my $max = 1024 * 1024; my $big = 'x' x (1024 * 1024 - 250); ok(length($big) > 512 * 1024); ok(length($big) < 1024 * 1024); # test that an even bigger value is rejected while we're here my $too_big = $big . $big . $big; my $len = length($too_big); print $sock "set too_big 0 0 $len\r\n$too_big\r\n"; is(scalar <$sock>, "SERVER_ERROR object too large for cache\r\n", "too_big not stored"); # set the big value my $len = length($big); print $sock "set big 0 0 $len\r\n$big\r\n"; is(scalar <$sock>, "STORED\r\n", "stored big"); mem_get_is($sock, "big", $big); # no evictions yet my $stats = mem_stats($sock); is($stats->{"evictions"}, "0", "no evictions to start"); # set many big items, enough to get evictions for (my $i = 0; $i < 100; $i++) { print $sock "set item_$i 0 0 $len\r\n$big\r\n"; is(scalar <$sock>, "STORED\r\n", "stored item_$i"); } # some evictions should have happened my $stats = mem_stats($sock); my $evictions = int($stats->{"evictions"}); ok($evictions == 37, "some evictions happened"); # the first big value should be gone mem_get_is($sock, "big", undef); # the earliest items should be gone too for (my $i = 0; $i < $evictions - 1; $i++) { mem_get_is($sock, "item_$i", undef); } # check that the non-evicted are the right ones for (my $i = $evictions - 1; $i < $evictions + 4; $i++) { mem_get_is($sock, "item_$i", $big); } memcached-1.5.22/t/flags.t0000775000175000017500000000071713461170555012175 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 8; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; # set foo (and should get it) for my $flags (0, 123, 2**16-1, 2**31) { print $sock "set foo $flags 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is({ sock => $sock, flags => $flags }, "foo", "fooval", "got flags $flags back"); } memcached-1.5.22/t/client_key.pem0000664000175000017500000000156713575022500013536 00000000000000-----BEGIN RSA PRIVATE KEY----- MIICXAIBAAKBgQCmpCQZFJ6WCj64FHvpBMqg72Ea5mOEUTQQWJ7yb50nZOWpMgLi +/DFR9Cyj/QZcSre+N6uywtBz8x2Y5tKmxJQX7m0/OP9BYV/pxyt7NFAcPpMUYij 1OVJsnJ8Lk3FAK5AlhWENF+ZdXtqANPsoH2C03GjeczUTDtQSdCbJ+QLywIDAQAB AoGAXFVMrxzutiIdGHA5LCb8g2m/+2C2uYUo/PmtsJVJlZ9hZXuRf+WrRhSBvb7n uQUshPmOoXld1mxmVR7h19fOsBbgKYqwDTlLZEnvdMVzaGiyeHiwDAvJgkkrK7pV kod5JIjFd+UMho6+Qn3K7tzfttLze5xuBPNrMH3q2b8LgBECQQDbJVyF+gLk4gmd BBInYl0BHPjYmGW9xfHfn0fhkKhQCZ9eNzH5Wbk1D8yqdlfmJ5nI8FkLQunzwzgv P51m+USJAkEAwqpaJoVOaaRfmsgEXOsZop32DBCBHwOqOyqH9qMez8qfBPDHrUbA TMtJN/TqLhi4VEPeaHbioUdUybA+1MggswJALWEgNrId1U2lVflY1QT+Y1OfiCKO tux9eKQgG2p8IA7ODJF7bLoSqxU5eXcGHqfGpaB3n+hcT9j2Enqm2oL1mQJAIp53 D9ivPDxeQEGH/RpWYcQjyLt6qxBUytbifSs/RIbtRsynRhqKAl44tDnbF72PsnSr bfqOjU4JNyEf22mH3wJBAK1yx2ilG//KYHtcfFYtll0Rkkife+2It6bTMVdkeVCH o5iDnW/+VsJCawhS9jHLZAjors+A9iyjygl1O6zLZTE= -----END RSA PRIVATE KEY----- memcached-1.5.22/t/dyn-maxbytes.t0000664000175000017500000000504713461170555013523 00000000000000#!/usr/bin/perl # Test the 'stats items' evictions counters. use strict; use Test::More tests => 309; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached("-m 3 -o modern,slab_automove_window=3"); my $sock = $server->sock; my $value = "B"x66560; my $key = 0; # These aren't set to expire. for ($key = 0; $key < 40; $key++) { print $sock "set key$key 0 0 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key$key"); } my $stats = mem_stats($sock); my $evicted = $stats->{evictions}; isnt($evicted, "0", "check evicted"); # We're past the memory limit. Try adjusting maxbytes upward. $stats = mem_stats($sock, "settings"); my $pre_maxbytes = $stats->{"maxbytes"}; print $sock "cache_memlimit 8\r\n"; is(scalar <$sock>, "OK\r\n", "bumped maxbytes from 3m to 8m"); # Confirm maxbytes updated. $stats = mem_stats($sock, "settings"); isnt($stats->{"maxbytes"}, $pre_maxbytes, "stats settings maxbytes updated"); # Check for total_malloced increasing as new memory is added $stats = mem_stats($sock, "slabs"); my $t_malloc = $stats->{"total_malloced"}; print $sock "set toast 0 0 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored toast"); $stats = mem_stats($sock, "slabs"); cmp_ok($stats->{"total_malloced"}, '>', $t_malloc, "stats slabs total_malloced increased"); $stats = mem_stats($sock); my $new_evicted = $stats->{evictions}; cmp_ok($new_evicted, '==', $evicted, "no new evictions"); # Bump up to 16, fill a bit more, then delete everything. print $sock "cache_memlimit 16\r\n"; is(scalar <$sock>, "OK\r\n", "bumped maxbytes from 8m to 16m"); for (;$key < 150; $key++) { print $sock "set key$key 0 0 66560\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key$key"); } # Grab total_malloced after filling everything up. $stats = mem_stats($sock, "slabs"); $t_malloc = $stats->{"total_malloced"}; print $sock "cache_memlimit 8\r\n"; is(scalar <$sock>, "OK\r\n", "bumped maxbytes from 16m to 8m"); # Remove all of the keys, allowing the slab rebalancer to push pages toward # the global page pool. for ($key = 0; $key < 150; $key++) { print $sock "delete key$key\r\n"; like(scalar <$sock>, qr/(DELETED|NOT_FOUND)\r\n/, "deleted key$key"); } # If memory limit is lower, it should free those pages back to the OS. my $reduced = 0; for (my $tries = 0; $tries < 6; $tries++) { sleep 1; $stats = mem_stats($sock, "slabs"); $reduced = $stats->{"total_malloced"} if ($t_malloc > $stats->{"total_malloced"}); last if $reduced; } isnt($reduced, 0, "total_malloced reduced to $reduced"); memcached-1.5.22/t/slabhang.t0000664000175000017500000000337113461170555012654 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; plan skip_all => 'Test is flaky. Needs special hooks.'; plan tests => 74; # start up a server with 10 maximum connections my $server = new_memcached("-m 16 -o modern"); my $sock = $server->sock; my $hangsock = $server->new_sock; my $value = "B"x260144; my $key = 0; # disable the normal automover. print $sock "slabs automove 0\r\n"; is(scalar <$sock>, "OK\r\n", "automover disabled"); # These aren't set to expire. my $mget = ''; for ($key = 0; $key < 70; $key++) { $mget .= "key$key "; print $sock "set key$key 0 0 260144\r\n$value\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key$key"); } chop $mget; # Don't intend to read the results, need to fill the socket. print $hangsock "get $mget\r\n"; #sleep 8; my $stats = mem_stats($sock, "slabs"); my $source = 0; for my $key (keys %$stats) { if ($key =~ m/^(\d+):total_pages/) { my $sid = $1; if ($stats->{$key} > 10) { $source = $sid; last; } } } isnt($source, 0, "found the source slab: $source"); my $busy; my $tomove = 4; my $reassign = "slabs reassign $source 1\r\n"; while ($tomove) { $busy = 0; print $sock $reassign; my $res = scalar <$sock>; while ($res =~ m/^BUSY/) { if ($hangsock && $busy > 5) { # unjam the pipeline $hangsock->close; } last if ($busy > 10); sleep 1; $busy++; print $sock $reassign; $res = scalar <$sock>; } last if ($busy > 10); $tomove--; } ok($busy <= 10, "didn't time out moving pages"); $stats = mem_stats($sock); isnt($stats->{"slab_reassign_busy_deletes"}, "0", "deleted some busy items"); memcached-1.5.22/t/ssl_ports.t0000664000175000017500000000126113575022500013111 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; if (!enabled_tls_testing()) { plan skip_all => 'SSL testing is not enabled'; exit 0; } my $tcp_port = free_port(); my $ssl_port = free_port(); my $server = new_memcached("-l notls:127.0.0.1:$tcp_port,127.0.0.1:$ssl_port", $ssl_port); my $sock = $server->sock; # Make sure we can talk over SSL print $sock "set foo:123 0 0 16\r\nfoo set over SSL\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); #.. and TCP my $tcp_sock = IO::Socket::INET->new(PeerAddr => "127.0.0.1:$tcp_port"); mem_get_is($tcp_sock, "foo:123", "foo set over SSL"); done_testing() memcached-1.5.22/t/client_crt.pem0000664000175000017500000000724513575022500013535 00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 2 (0x2) Signature Algorithm: sha1WithRSAEncryption Issuer: CN=Test Root Certificate Authority, ST=CA, C=US/emailAddress=root@test.com, O=Test, OU=Test Department Validity Not Before: Jan 7 19:26:14 2019 GMT Not After : Jan 6 19:26:14 2024 GMT Subject: CN=client.test.com, ST=CA, C=US/emailAddress=root@client.test.com, O=Test Client, OU=Subunit of Test Organization Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (1024 bit) Modulus: 00:a6:a4:24:19:14:9e:96:0a:3e:b8:14:7b:e9:04: ca:a0:ef:61:1a:e6:63:84:51:34:10:58:9e:f2:6f: 9d:27:64:e5:a9:32:02:e2:fb:f0:c5:47:d0:b2:8f: f4:19:71:2a:de:f8:de:ae:cb:0b:41:cf:cc:76:63: 9b:4a:9b:12:50:5f:b9:b4:fc:e3:fd:05:85:7f:a7: 1c:ad:ec:d1:40:70:fa:4c:51:88:a3:d4:e5:49:b2: 72:7c:2e:4d:c5:00:ae:40:96:15:84:34:5f:99:75: 7b:6a:00:d3:ec:a0:7d:82:d3:71:a3:79:cc:d4:4c: 3b:50:49:d0:9b:27:e4:0b:cb Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE X509v3 Key Usage: Digital Signature, Non Repudiation, Key Encipherment X509v3 Subject Alternative Name: DNS:client.test.com, DNS:alt.client.test.com Signature Algorithm: sha1WithRSAEncryption 4f:11:ff:ed:05:e2:80:75:bd:e0:b3:f3:21:34:65:50:67:ff: 3c:45:88:58:58:77:9a:6e:f6:1e:74:f9:cb:02:e1:31:e0:52: d9:f9:63:e5:fb:01:d2:83:df:20:c1:77:a7:15:da:18:3a:e0: ea:e0:66:ab:41:21:9e:36:9c:36:28:1c:cb:20:43:94:94:e8: 9b:0d:3d:2a:ac:20:48:5b:b8:c0:45:0d:5c:30:91:be:ba:67: b9:f6:bd:64:08:ab:af:35:a1:db:dd:54:e8:32:c9:3a:95:34: 26:8f:a1:1b:a1:a2:32:47:a0:e0:a4:11:06:dc:d2:67:87:1a: 51:50:bc:09:26:e3:1c:e7:83:a5:69:48:92:6c:87:94:46:f6: b2:45:55:6f:5e:f2:6b:c8:9d:65:61:31:83:09:71:60:71:d5: 9c:44:65:27:f6:3f:fd:fb:40:30:47:02:b1:6f:5a:ff:7a:c5: 83:e0:80:52:53:a7:2c:24:71:51:81:df:3f:2f:1d:42:df:bc: 86:b4:0c:18:64:8d:33:a7:c0:e8:f2:9e:f5:0b:92:c4:4d:f7: 4b:2f:13:8d:81:25:f3:47:f5:72:71:c2:62:3d:36:09:3c:ec: d1:15:6d:15:77:28:c6:de:f9:73:5d:5b:a0:a4:0f:f2:50:a0: 00:20:87:fa -----BEGIN CERTIFICATE----- MIIDZTCCAk2gAwIBAgIBAjANBgkqhkiG9w0BAQUFADCBizEoMCYGA1UEAwwfVGVz dCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eTELMAkGA1UECAwCQ0ExCzAJBgNV BAYTAlVTMRwwGgYJKoZIhvcNAQkBFg1yb290QHRlc3QuY29tMQ0wCwYDVQQKDARU ZXN0MRgwFgYDVQQLDA9UZXN0IERlcGFydG1lbnQwHhcNMTkwMTA3MTkyNjE0WhcN MjQwMTA2MTkyNjE0WjCBljEYMBYGA1UEAwwPY2xpZW50LnRlc3QuY29tMQswCQYD VQQIDAJDQTELMAkGA1UEBhMCVVMxIzAhBgkqhkiG9w0BCQEWFHJvb3RAY2xpZW50 LnRlc3QuY29tMRQwEgYDVQQKDAtUZXN0IENsaWVudDElMCMGA1UECwwcU3VidW5p dCBvZiBUZXN0IE9yZ2FuaXphdGlvbjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC gYEApqQkGRSelgo+uBR76QTKoO9hGuZjhFE0EFie8m+dJ2TlqTIC4vvwxUfQso/0 GXEq3vjerssLQc/MdmObSpsSUF+5tPzj/QWFf6ccrezRQHD6TFGIo9TlSbJyfC5N xQCuQJYVhDRfmXV7agDT7KB9gtNxo3nM1Ew7UEnQmyfkC8sCAwEAAaNLMEkwCQYD VR0TBAIwADALBgNVHQ8EBAMCBeAwLwYDVR0RBCgwJoIPY2xpZW50LnRlc3QuY29t ghNhbHQuY2xpZW50LnRlc3QuY29tMA0GCSqGSIb3DQEBBQUAA4IBAQBPEf/tBeKA db3gs/MhNGVQZ/88RYhYWHeabvYedPnLAuEx4FLZ+WPl+wHSg98gwXenFdoYOuDq 4GarQSGeNpw2KBzLIEOUlOibDT0qrCBIW7jARQ1cMJG+ume59r1kCKuvNaHb3VTo Msk6lTQmj6EboaIyR6DgpBEG3NJnhxpRULwJJuMc54OlaUiSbIeURvayRVVvXvJr yJ1lYTGDCXFgcdWcRGUn9j/9+0AwRwKxb1r/esWD4IBSU6csJHFRgd8/Lx1C37yG tAwYZI0zp8Do8p71C5LETfdLLxONgSXzR/VyccJiPTYJPOzRFW0VdyjG3vlzXVug pA/yUKAAIIf6 -----END CERTIFICATE----- memcached-1.5.22/t/incrdecr.t0000775000175000017500000000415513250303500012652 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 23; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; # Bug 21 print $sock "set bug21 0 0 19\r\n9223372036854775807\r\n"; is(scalar <$sock>, "STORED\r\n", "stored text"); print $sock "incr bug21 1\r\n"; is(scalar <$sock>, "9223372036854775808\r\n", "bug21 incr 1"); print $sock "incr bug21 1\r\n"; is(scalar <$sock>, "9223372036854775809\r\n", "bug21 incr 2"); print $sock "decr bug21 1\r\n"; is(scalar <$sock>, "9223372036854775808\r\n", "bug21 decr"); print $sock "set num 0 0 1\r\n1\r\n"; is(scalar <$sock>, "STORED\r\n", "stored num"); mem_get_is($sock, "num", 1, "stored 1"); print $sock "incr num 1\r\n"; is(scalar <$sock>, "2\r\n", "+ 1 = 2"); mem_get_is($sock, "num", 2); print $sock "incr num 8\r\n"; is(scalar <$sock>, "10\r\n", "+ 8 = 10"); mem_get_is($sock, "num", 10); print $sock "decr num 1\r\n"; is(scalar <$sock>, "9\r\n", "- 1 = 9"); print $sock "decr num 9\r\n"; is(scalar <$sock>, "0\r\n", "- 9 = 0"); print $sock "decr num 5\r\n"; is(scalar <$sock>, "0\r\n", "- 5 = 0"); printf $sock "set num 0 0 10\r\n4294967296\r\n"; is(scalar <$sock>, "STORED\r\n", "stored 2**32"); print $sock "incr num 1\r\n"; is(scalar <$sock>, "4294967297\r\n", "4294967296 + 1 = 4294967297"); printf $sock "set num 0 0 %d\r\n18446744073709551615\r\n", length("18446744073709551615"); is(scalar <$sock>, "STORED\r\n", "stored 2**64-1"); print $sock "incr num 1\r\n"; is(scalar <$sock>, "0\r\n", "(2**64 - 1) + 1 = 0"); print $sock "decr bogus 5\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "can't decr bogus key"); print $sock "decr incr 5\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "can't incr bogus key"); print $sock "set bigincr 0 0 1\r\n0\r\n"; is(scalar <$sock>, "STORED\r\n", "stored bigincr"); print $sock "incr bigincr 18446744073709551610\r\n"; is(scalar <$sock>, "18446744073709551610\r\n"); print $sock "set text 0 0 2\r\nhi\r\n"; is(scalar <$sock>, "STORED\r\n", "stored hi"); print $sock "incr text 1\r\n"; is(scalar <$sock>, "CLIENT_ERROR cannot increment or decrement non-numeric value\r\n", "hi - 1 = 0"); memcached-1.5.22/t/misbehave.t0000775000175000017500000000117713575022500013035 00000000000000#!/usr/bin/perl use strict; use Test::More; use FindBin qw($Bin); use Socket qw(MSG_PEEK MSG_DONTWAIT); use lib "$Bin/lib"; use MemcachedTest; if (!enabled_tls_testing() && supports_drop_priv()) { plan tests => 1; } else { plan skip_all => 'Privilege drop not supported'; exit 0; } my $server = new_memcached('-o drop_privileges'); my $sock = $server->sock; print $sock "misbehave\r\n"; sleep(1); # check if the socket is dead now my $buff; my $ret = recv($sock, $buff, 1, MSG_PEEK | MSG_DONTWAIT); # ret = 0 means read 0 bytes, which means a closed socket ok($ret == 0, "did not allow misbehaving"); $server->DESTROY(); memcached-1.5.22/t/issue_67.t0000664000175000017500000000561113461170555012540 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 24; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use Carp qw(croak); use Socket qw(sockaddr_in INADDR_ANY PF_INET SOCK_STREAM); use Cwd; my $builddir = getcwd; $ENV{'MEMCACHED_PORT_FILENAME'} = "/tmp/ports.$$"; sub read_ports { my %rv = (); open(my $f, "/tmp/ports.$$") || die("Can't open ports file."); while(<$f>) { my ($type, $port) = split(/:\s+/); $rv{$type} = $port + 0; } unlink "/tmp/ports.$$"; return %rv; } sub validate_port { my ($name, $got, $expected) = @_; # diag "Wanted $expected, got $got"; if ($expected == -1) { ok(!defined($got), "$name expected no port, got $got"); } elsif ($expected == 0) { ok(defined($got) && $got != 11211, "$name expected random port (got $got)"); } else { is($got, $expected, "$name"); } } sub skip_if_default_addr_in_use(&) { my ($block) = @_; socket(my $socket, PF_INET, SOCK_STREAM, 0) or die $!; my $addr_in_use = !bind($socket, sockaddr_in(11211, INADDR_ANY)); close($socket); SKIP: { skip 'Default address is in use. Do you have a running instance?', 2 if $addr_in_use; return $block->(); } } sub run_server { my ($args) = @_; my $exe = "$builddir/memcached-debug"; croak("memcached binary doesn't exist. Haven't run 'make' ?\n") unless -e $exe; my $childpid = fork(); my $root = ''; $root = "-u root" if ($< == 0); # test build requires more privileges $args .= " -o relaxed_privileges"; my $cmd = "$builddir/timedrun 10 $exe $root $args"; unless($childpid) { exec $cmd; exit; # NOTREACHED } for (1..20) { if (-f "/tmp/ports.$$") { return Memcached::Handle->new(pid => $childpid); } select undef, undef, undef, 0.10; } croak "Failed to start server."; } sub when { my ($name, $params, $expected_tcp, $expected_udp) = @_; my $server = run_server($params); my %ports = read_ports(); validate_port($name, $ports{'TCP INET'}, $expected_tcp); validate_port($name, $ports{'UDP INET'}, $expected_udp); } skip_if_default_addr_in_use { when('no arguments', '', 11211, -1) }; when('specifying tcp port', '-p 11212', 11212, -1); when('specifying udp port', '-U 11222', 11222, 11222); when('specifying tcp ephemeral port', '-p -1', 0, -1); when('specifying udp ephemeral port', '-U -1', 0, 0); when('tcp port disabled', '-p 0', -1, -1); skip_if_default_addr_in_use { when('udp port disabled', '-U 0', 11211, -1) }; when('specifying tcp and udp ports', '-p 11232 -U 11233', 11232, 11233); when('specifying tcp and disabling udp', '-p 11242 -U 0', 11242, -1); when('specifying udp and disabling tcp', '-p -1 -U 11252', 0, 11252); when('specifying tcp and ephemeral udp', '-p 11262 -U -1', 11262, 0); when('specifying udp and ephemeral tcp', '-p -1 -U 11272', 0, 11272); memcached-1.5.22/t/whitespace.t0000775000175000017500000000311713611703161013222 00000000000000#!/usr/bin/perl use strict; use FindBin qw($Bin); our @files; BEGIN { chdir "$Bin/.." or die; unless (-d "$Bin/../.git") { use Test::More; plan skip_all => "Skipping test because this does not appear to be a memcached git working directory"; exit 0; } my @exempted = qw(Makefile.am ChangeLog doc/Makefile.am README README.md compile_commands.json); push(@exempted, glob("doc/*.xml")); push(@exempted, glob("doc/*.full")); push(@exempted, glob("doc/xml2rfc/*.xsl")); push(@exempted, glob("doc/xml2rfc/*.dtd")); push(@exempted, glob("m4/*backport*m4")); push(@exempted, glob("*.orig")); push(@exempted, glob(".*.swp")); my %exempted_hash = map { $_ => 1 } @exempted; my @stuff = split /\0/, `git ls-files -z -c -m -o --exclude-standard`; @files = grep { ! $exempted_hash{$_} } @stuff; # We won't find any files if git isn't installed. If git isn't # installed, they're probably not doing any useful development, or # at the very least am will clean up whitespace when we receive # their patch. unless (@files) { use Test::More; plan skip_all => "Skipping tests probably because you don't have git."; exit 0; } } use Test::More tests => scalar(@files); foreach my $f (@files) { open(my $fh, $f) or die "Cannot open file $f: $!"; my $before = do { local $/; <$fh>; }; close ($fh); my $after = $before; $after =~ s/\t/ /g; $after =~ s/ +$//mg; $after .= "\n" unless $after =~ /\n$/; ok ($after eq $before, "$f (see devtools/clean-whitespace.pl)"); } memcached-1.5.22/t/extstore-buckets.t0000664000175000017500000000337613461170555014415 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use Data::Dumper qw/Dumper/; my $ext_path; if (!supports_extstore()) { plan skip_all => 'extstore not enabled'; exit 0; } $ext_path = "/tmp/extstore.$$"; my $server = new_memcached("-m 256 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0,ext_path=$ext_path:64m,ext_low_ttl=60,slab_automove=1"); my $sock = $server->sock; my $value; { my @chars = ("C".."Z"); for (1 .. 20000) { $value .= $chars[rand @chars]; } } # fill some larger objects { # interleave sets with 0 ttl vs long ttl's. my $keycount = 1200; for (1 .. $keycount) { print $sock "set nfoo$_ 0 0 20000 noreply\r\n$value\r\n"; print $sock "set lfoo$_ 0 5 20000 noreply\r\n$value\r\n"; } # wait for a flush sleep 10; print $sock "lru_crawler crawl all\r\n"; <$sock>; sleep 2; # fetch mem_get_is($sock, "nfoo1", $value); # check extstore counters my $stats = mem_stats($sock); cmp_ok($stats->{extstore_page_allocs}, '>', 0, 'at least one page allocated'); cmp_ok($stats->{extstore_objects_written}, '>', $keycount / 2, 'some objects written'); cmp_ok($stats->{extstore_bytes_written}, '>', length($value) * 2, 'some bytes written'); cmp_ok($stats->{get_extstore}, '>', 0, 'one object was fetched'); cmp_ok($stats->{extstore_objects_read}, '>', 0, 'one object read'); cmp_ok($stats->{extstore_bytes_read}, '>', length($value), 'some bytes read'); cmp_ok($stats->{extstore_page_reclaims}, '>', 1, 'at least two pages reclaimed'); } done_testing(); END { unlink $ext_path if $ext_path; } memcached-1.5.22/t/00-startup.t0000775000175000017500000000412313611703161013003 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 21; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; eval { my $server = new_memcached(); ok($server, "started the server"); }; is($@, '', 'Basic startup works'); eval { my $server = new_memcached("-l fooble"); }; ok($@, "Died with illegal -l args"); eval { my $server = new_memcached("-l 127.0.0.1"); }; is($@,'', "-l 127.0.0.1 works"); eval { my $server = new_memcached('-C'); my $stats = mem_stats($server->sock, 'settings'); is('no', $stats->{'cas_enabled'}); }; is($@, '', "-C works"); eval { my $server = new_memcached('-b 8675'); my $stats = mem_stats($server->sock, 'settings'); is('8675', $stats->{'tcp_backlog'}); }; is($@, '', "-b works"); foreach my $val ('auto', 'ascii') { eval { my $server = new_memcached("-B $val"); my $stats = mem_stats($server->sock, 'settings'); ok($stats->{'binding_protocol'} =~ /$val/, "$val works"); }; is($@, '', "$val works"); } # For the binary test, we just verify it starts since we don't have an easy bin client. eval { my $server = new_memcached("-B binary"); }; is($@, '', "binary works"); eval { my $server = new_memcached("-vv -B auto"); }; is($@, '', "auto works"); eval { my $server = new_memcached("-vv -B ascii"); }; is($@, '', "ascii works"); # For the binary test, we just verify it starts since we don't have an easy bin client. eval { my $server = new_memcached("-vv -B binary"); }; is($@, '', "binary works"); # Should blow up with something invalid. eval { my $server = new_memcached("-B http"); }; ok($@, "Died with illegal -B arg."); # Maximum connections must be greater than 0. eval { my $server = new_memcached("-c 0"); }; ok($@, "Died with invalid maximum connections 0."); eval { my $server = new_memcached("-c -1"); }; ok($@, "Died with invalid maximum connections -1."); # Should not allow -t 0 eval { my $server = new_memcached("-t 0"); }; ok($@, "Died with illegal 0 thread count"); { my $exit_code = run_help(); is($exit_code, 0, "Help defaults are fine."); }; memcached-1.5.22/t/issue_108.t0000664000175000017500000000125713250303500012576 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 4; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $key = "del_key"; print $sock "add $key 0 0 1\r\nx\r\n"; is (scalar <$sock>, "STORED\r\n", "Added a key"); print $sock "delete $key 0\r\n"; is (scalar <$sock>, "DELETED\r\n", "Properly deleted with 0"); print $sock "add $key 0 0 1\r\nx\r\n"; is (scalar <$sock>, "STORED\r\n", "Added again a key"); print $sock "delete $key 0 noreply\r\n"; # will not reply, but a subsequent add will succeed print $sock "add $key 0 0 1\r\nx\r\n"; is (scalar <$sock>, "STORED\r\n", "Add succeeded after quiet deletion."); memcached-1.5.22/t/dash-M.t0000664000175000017500000000132513461170555012203 00000000000000#!/usr/bin/perl use strict; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached('-M -m 2'); my $sock = $server->sock; my $value = "B" x 8192; my $vallen = length($value); my $resp = "STORED\r\n"; my $key = 0; while($resp eq "STORED\r\n") { print $sock "set dash$key 0 0 $vallen\r\n$value\r\n"; $key++; $resp = scalar <$sock>; } my $max_stored = $key - 1; plan tests => $max_stored + 1; print $sock "set dash$key 0 0 $vallen\r\n$value\r\n"; is(scalar <$sock>, "SERVER_ERROR out of memory storing object\r\n", "failed to add another one."); for($key = 0; $key < $max_stored; $key++) { mem_get_is $sock, "dash$key", $value, "Failed at dash$key"; } memcached-1.5.22/t/issue_163.t0000664000175000017500000000205413461170555012613 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 7; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $value1 = "A"x66560; my $value2 = "B"x66570; print $sock "set key 0 1 66560\r\n$value1\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key"); my $stats = mem_stats($sock, "slabs"); my $requested = $stats->{"31:mem_requested"}; isnt ($requested, "0", "We should have requested some memory"); sleep(3); print $sock "set key 0 0 66570\r\n$value2\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key"); my $stats = mem_stats($sock, "items"); my $reclaimed = $stats->{"items:31:reclaimed"}; is ($reclaimed, "1", "Objects should be reclaimed"); print $sock "delete key\r\n"; is (scalar <$sock>, "DELETED\r\n", "deleted key"); print $sock "set key 0 0 66560\r\n$value1\r\n"; is (scalar <$sock>, "STORED\r\n", "stored key"); my $stats = mem_stats($sock, "slabs"); my $requested2 = $stats->{"31:mem_requested"}; is ($requested2, $requested, "we've not allocated and freed the same amount"); memcached-1.5.22/t/issue_192.t0000664000175000017500000000104413461170555012613 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 2; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; ok($server->new_sock, "opened new socket"); print $sock "\x80\x12\x00\x01\x08\x00\x00\x00\xff\xff\xff\xe8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x01\x00\x00\x00\x00\x00\x00\x00\x00\x000\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"; sleep 0.5; ok($server->new_sock, "failed to open new socket"); memcached-1.5.22/t/cacert.pem0000664000175000017500000000246613611710715012653 00000000000000-----BEGIN CERTIFICATE----- MIIDqzCCApOgAwIBAgIJAOFc3ZmVKolRMA0GCSqGSIb3DQEBBQUAMIGLMSgwJgYD VQQDDB9UZXN0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MQswCQYDVQQIDAJD QTELMAkGA1UEBhMCVVMxHDAaBgkqhkiG9w0BCQEWDXJvb3RAdGVzdC5jb20xDTAL BgNVBAoMBFRlc3QxGDAWBgNVBAsMD1Rlc3QgRGVwYXJ0bWVudDAeFw0xOTAxMDcx ODIzMjlaFw0yNDAxMDYxODIzMjlaMIGLMSgwJgYDVQQDDB9UZXN0IFJvb3QgQ2Vy dGlmaWNhdGUgQXV0aG9yaXR5MQswCQYDVQQIDAJDQTELMAkGA1UEBhMCVVMxHDAa BgkqhkiG9w0BCQEWDXJvb3RAdGVzdC5jb20xDTALBgNVBAoMBFRlc3QxGDAWBgNV BAsMD1Rlc3QgRGVwYXJ0bWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC ggEBAJQcDfenVed0uJCykaWowL2qGUs9e5gyjGo9URoNhI5LDYSSq3ebxZGfbi1T BcC7oL5OI/B0lCShPR9sJjKFkQ60vs43ltmkTLNSGoKLnxXMlBjdpCxweDDAMiF3 p/vCG2hUa7auOkMWMYIkM81rcQsRB0qj0ilt3zcTsS860oKGzNrtPeAcz3KxbYWI nJEhQVy3S4U59b7mm3cGz3/3m2NUjn+b8sA7J8F9K5mnFkCUCRva6zte6qmL2ruH sGRav9ICLGxDqiJoic6Y2ReffgU77RDJO+sTuJme+VeTDE77vBIHvCVCSG+e0RSs L+6nQYDtjHH5bLgoSXq9D3hgxEsCAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkq hkiG9w0BAQUFAAOCAQEARWcYJvkleq+qfOPPI1eQAuBf5OZwz8mthsBvMfsQMCLk +wSxdlJvJyvPoFgZBv8YbTde2b40lQe94gTsGDUXGEkOsERrjt/jqHxO9X8n7a+z M/okeSSYnam0Zcl1M9sa5L7BVXbGh/sE9j/bXrAhz64np5P773dZTLTPYjBf2Grp NheCsGDtJbegJqn7pp5MfAKcyzLxnZAE0cilSVKZB7R3urISJVdwiRtkprJL7IwQ oIu+XhUgdZbx7TQQnjTkq3COSIIof5+5oqFnhzTqfSgi/06dWWvCwl17Mz+vCMZ9 1MA8L4cR+iNdJYlCQPbk30laJx1akfqnpv7qTXq7nQ== -----END CERTIFICATE----- memcached-1.5.22/t/item_size_max.t0000664000175000017500000000236213461170555013731 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 7; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $stats = mem_stats($sock, ' settings'); # Ensure default still works. is($stats->{item_size_max}, 1024 * 1024); $server->stop(); # Should die. eval { $server = new_memcached('-I 1000'); }; ok($@ && $@ =~ m/^Failed/, "Shouldn't start with < 1k item max"); eval { $server = new_memcached('-I 256m'); }; ok($@ && $@ =~ m/^Failed/, "Shouldn't start with > 128m item max"); # Minimum. $server = new_memcached('-I 1024 -o slab_chunk_max=1024'); my $stats = mem_stats($server->sock, ' settings'); is($stats->{item_size_max}, 1024); $server->stop(); # Reasonable but unreasonable. $server = new_memcached('-I 2097152'); my $stats = mem_stats($server->sock, ' settings'); is($stats->{item_size_max}, 2097152); $server->stop(); # Suffix kilobytes. $server = new_memcached('-I 512k -o slab_chunk_max=16384'); my $stats = mem_stats($server->sock, ' settings'); is($stats->{item_size_max}, 524288); $server->stop(); # Suffix megabytes. $server = new_memcached('-m 256 -I 32m'); my $stats = mem_stats($server->sock, ' settings'); is($stats->{item_size_max}, 33554432); $server->stop(); memcached-1.5.22/t/noreply.t0000664000175000017500000000227713575022500012561 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 9; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; # Test that commands can take 'noreply' parameter. print $sock "flush_all noreply\r\n"; print $sock "flush_all 0 noreply\r\n"; print $sock "verbosity 0 noreply\r\n"; print $sock "add noreply:foo 0 0 1 noreply\r\n1\r\n"; mem_get_is($sock, "noreply:foo", "1"); print $sock "set noreply:foo 0 0 1 noreply\r\n2\r\n"; mem_get_is($sock, "noreply:foo", "2"); print $sock "replace noreply:foo 0 0 1 noreply\r\n3\r\n"; mem_get_is($sock, "noreply:foo", "3"); print $sock "append noreply:foo 0 0 1 noreply\r\n4\r\n"; mem_get_is($sock, "noreply:foo", "34"); print $sock "prepend noreply:foo 0 0 1 noreply\r\n5\r\n"; my @result = mem_gets($sock, "noreply:foo"); ok($result[1] eq "534"); print $sock "cas noreply:foo 0 0 1 $result[0] noreply\r\n6\r\n"; mem_get_is($sock, "noreply:foo", "6"); print $sock "incr noreply:foo 3 noreply\r\n"; mem_get_is($sock, "noreply:foo", "9"); print $sock "decr noreply:foo 2 noreply\r\n"; mem_get_is($sock, "noreply:foo", "7"); print $sock "delete noreply:foo noreply\r\n"; mem_get_is($sock, "noreply:foo"); memcached-1.5.22/t/ssl_verify_modes.t0000664000175000017500000000102413575022500014432 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; use Cwd; if (!enabled_tls_testing()) { plan skip_all => 'SSL testing is not enabled'; exit 0; } my $ca_crt = getcwd() . "/t/" . MemcachedTest::CA_CRT; my $server = new_memcached("-o ssl_verify_mode=2 -o ssl_ca_cert=$ca_crt"); # just using stats to make sure everything is working fine. my $stats = mem_stats($server->sock); is($stats->{accepting_conns}, 1, "client cert is verified"); done_testing(); memcached-1.5.22/t/issue_29.t0000664000175000017500000000116713250303500012520 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 4; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; print $sock "set issue29 0 0 0\r\n\r\n"; is (scalar <$sock>, "STORED\r\n", "stored issue29"); my $first_stats = mem_stats($sock, "slabs"); my $first_used = $first_stats->{"1:used_chunks"}; is(1, $first_used, "Used one"); print $sock "set issue29_b 0 0 0\r\n\r\n"; is (scalar <$sock>, "STORED\r\n", "stored issue29_b"); my $second_stats = mem_stats($sock, "slabs"); my $second_used = $second_stats->{"1:used_chunks"}; is(2, $second_used, "Used two") memcached-1.5.22/t/touch.t0000775000175000017500000000073113461170555012217 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 4; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; # set foo (and should get it) print $sock "set foo 0 2 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); mem_get_is($sock, "foo", "fooval"); # touch it print $sock "touch foo 10\r\n"; is(scalar <$sock>, "TOUCHED\r\n", "touched foo"); sleep 2; mem_get_is($sock, "foo", "fooval"); memcached-1.5.22/t/lru-crawler.t0000664000175000017500000000523613615376577013353 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More tests => 222; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached('-m 32 -o no_modern'); { my $stats = mem_stats($server->sock, ' settings'); is($stats->{lru_crawler}, "no"); } my $sock = $server->sock; # Fill a slab a bit. # Some immortal items, some long expiring items, some short expiring items. # Done so the immortals end up at the tail. for (1 .. 30) { print $sock "set ifoo$_ 0 0 2\r\nok\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); } for (1 .. 30) { print $sock "set lfoo$_ 0 3600 2\r\nok\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); } for (1 .. 30) { print $sock "set sfoo$_ 0 1 2\r\nok\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); } { my $slabs = mem_stats($sock, "slabs"); is($slabs->{"1:used_chunks"}, 90, "slab1 has 90 used chunks"); } sleep 3; print $sock "lru_crawler enable\r\n"; is(scalar <$sock>, "OK\r\n", "enabled lru crawler"); { my $stats = mem_stats($server->sock, ' settings'); is($stats->{lru_crawler}, "yes"); } print $sock "lru_crawler crawl 1\r\n"; is(scalar <$sock>, "OK\r\n", "kicked lru crawler"); while (1) { my $stats = mem_stats($sock); last unless $stats->{lru_crawler_running}; sleep 1; } { my $slabs = mem_stats($sock, "slabs"); is($slabs->{"1:used_chunks"}, 60, "slab1 now has 60 used chunks"); my $items = mem_stats($sock, "items"); is($items->{"items:1:crawler_reclaimed"}, 30, "slab1 has 30 reclaims"); } # Check that crawler metadump works correctly. { print $sock "lru_crawler metadump all\r\n"; my $count = 0; while (<$sock>) { last if /^(\.|END)/; /^(key=) (\S+).*([^\r\n]+)/; $count++; } is ($count, 60); } for (1 .. 30) { mem_get_is($sock, "ifoo$_", "ok"); mem_get_is($sock, "lfoo$_", "ok"); mem_get_is($sock, "sfoo$_", undef); } print $sock "lru_crawler disable\r\n"; is(scalar <$sock>, "OK\r\n", "disabled lru crawler"); { my $stats = mem_stats($server->sock, ' settings'); is($stats->{lru_crawler}, "no"); } $server->stop; # Test initializing crawler from starttime. $server = new_memcached('-m 32 -o no_modern,lru_crawler'); $sock = $server->sock; for (1 .. 30) { print $sock "set sfoo$_ 0 1 2\r\nok\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); } sleep 3; print $sock "lru_crawler crawl 1\r\n"; is(scalar <$sock>, "OK\r\n", "kicked lru crawler"); while (1) { my $stats = mem_stats($sock); last unless $stats->{lru_crawler_running}; sleep 1; } { my $slabs = mem_stats($sock, "slabs"); is($slabs->{"1:used_chunks"}, 0, "slab1 now has 0 used chunks"); } memcached-1.5.22/t/stats-detail.t0000664000175000017500000000410513250303500013447 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 24; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $expire; print $sock "stats detail dump\r\n"; is(scalar <$sock>, "END\r\n", "verified empty stats at start"); print $sock "stats detail on\r\n"; is(scalar <$sock>, "OK\r\n", "detail collection turned on"); print $sock "set foo:123 0 0 6\r\nfooval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored foo"); print $sock "stats detail dump\r\n"; is(scalar <$sock>, "PREFIX foo get 0 hit 0 set 1 del 0\r\n", "details after set"); is(scalar <$sock>, "END\r\n", "end of details"); mem_get_is($sock, "foo:123", "fooval"); print $sock "stats detail dump\r\n"; is(scalar <$sock>, "PREFIX foo get 1 hit 1 set 1 del 0\r\n", "details after get with hit"); is(scalar <$sock>, "END\r\n", "end of details"); mem_get_is($sock, "foo:124", undef); print $sock "stats detail dump\r\n"; is(scalar <$sock>, "PREFIX foo get 2 hit 1 set 1 del 0\r\n", "details after get without hit"); is(scalar <$sock>, "END\r\n", "end of details"); print $sock "delete foo:125\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "sent delete command"); print $sock "stats detail dump\r\n"; is(scalar <$sock>, "PREFIX foo get 2 hit 1 set 1 del 1\r\n", "details after delete"); is(scalar <$sock>, "END\r\n", "end of details"); print $sock "stats reset\r\n"; is(scalar <$sock>, "RESET\r\n", "stats cleared"); print $sock "stats detail dump\r\n"; is(scalar <$sock>, "END\r\n", "empty stats after clear"); mem_get_is($sock, "foo:123", "fooval"); print $sock "stats detail dump\r\n"; is(scalar <$sock>, "PREFIX foo get 1 hit 1 set 0 del 0\r\n", "details after clear and get"); is(scalar <$sock>, "END\r\n", "end of details"); print $sock "stats detail off\r\n"; is(scalar <$sock>, "OK\r\n", "detail collection turned off"); mem_get_is($sock, "foo:124", undef); mem_get_is($sock, "foo:123", "fooval"); print $sock "stats detail dump\r\n"; is(scalar <$sock>, "PREFIX foo get 1 hit 1 set 0 del 0\r\n", "details after stats turned off"); is(scalar <$sock>, "END\r\n", "end of details"); memcached-1.5.22/t/slabs_reassign.t0000664000175000017500000000465713461170555014104 00000000000000#!/usr/bin/perl use strict; use warnings; use Test::More tests => 130; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; # Enable manual slab reassign, cap at 6 slabs my $server = new_memcached('-o slab_reassign -m 4'); my $stats = mem_stats($server->sock, ' settings'); is($stats->{slab_reassign}, "yes"); my $sock = $server->sock; # Fill a largeish slab until it evicts (honors the -m 6) my $bigdata = 'x' x 70000; # slab 31 for (1 .. 60) { print $sock "set bfoo$_ 0 0 70000\r\n", $bigdata, "\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); } # Fill a smaller slab until it evicts my $smalldata = 'y' x 20000; # slab 25 for (1 .. 60) { print $sock "set sfoo$_ 0 0 20000\r\n", $smalldata, "\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); } my $items_before = mem_stats($sock, "items"); isnt($items_before->{"items:31:evicted"}, 0, "slab 31 evicted is nonzero"); isnt($items_before->{"items:25:evicted"}, 0, "slab 25 evicted is nonzero"); my $slabs_before = mem_stats($sock, "slabs"); # Move a large slab to the smaller slab print $sock "slabs reassign 31 25\r\n"; is(scalar <$sock>, "OK\r\n", "slab rebalancer started"); # Still working out how/if to signal the thread. For now, just sleep. sleep 2; # Check that stats counters increased my $slabs_after = mem_stats($sock, "slabs"); $stats = mem_stats($sock); isnt($stats->{slabs_moved}, 0, "slabs moved is nonzero"); # Check that slab stats reflect the change ok($slabs_before->{"31:total_pages"} != $slabs_after->{"31:total_pages"}, "slab 31 pagecount changed"); ok($slabs_before->{"25:total_pages"} != $slabs_after->{"25:total_pages"}, "slab 25 pagecount changed"); # Try to move another slab, see that you can move two in a row print $sock "slabs reassign 31 25\r\n"; like(scalar <$sock>, qr/^OK/, "Cannot re-run against class with empty space"); # Try to move a page backwards. Should complain that source class isn't "safe" # to move from. # TODO: Wait until the above command completes, then try to move it back? # Seems pointless... #print $sock "slabs reassign 25 31\r\n"; #like(scalar <$sock>, qr/^UNSAFE/, "Cannot move an unsafe slab back"); # Try to insert items into both slabs print $sock "set bfoo51 0 0 70000\r\n", $bigdata, "\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); print $sock "set sfoo51 0 0 20000\r\n", $smalldata, "\r\n"; is(scalar <$sock>, "STORED\r\n", "stored key"); # Do need to come up with better automated tests for this. memcached-1.5.22/t/cas.t0000664000175000017500000001113513461170555011640 00000000000000#!/usr/bin/perl use strict; use Test::More tests => 43; use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; my $server = new_memcached(); my $sock = $server->sock; my $sock2 = $server->new_sock; my @result; my @result2; ok($sock != $sock2, "have two different connections open"); sub check_args { my ($line, $name) = @_; my $svr = new_memcached(); my $s = $svr->sock; print $s $line; is(scalar <$s>, "CLIENT_ERROR bad command line format\r\n", $name); undef $svr; } check_args "cas bad blah 0 0 0\r\n\r\n", "bad flags"; check_args "cas bad 0 blah 0 0\r\n\r\n", "bad exp"; check_args "cas bad 0 0 blah 0\r\n\r\n", "bad cas"; check_args "cas bad 0 0 0 blah\r\n\r\n", "bad size"; # gets foo (should not exist) print $sock "gets foo\r\n"; is(scalar <$sock>, "END\r\n", "gets failed"); # set foo print $sock "set foo 0 0 6\r\nbarval\r\n"; is(scalar <$sock>, "STORED\r\n", "stored barval"); # gets foo and verify identifier exists @result = mem_gets($sock, "foo"); mem_gets_is($sock,$result[0],"foo","barval"); # cas fail print $sock "cas foo 0 0 6 123\r\nbarva2\r\n"; is(scalar <$sock>, "EXISTS\r\n", "cas failed for foo"); # gets foo - success @result = mem_gets($sock, "foo"); mem_gets_is($sock,$result[0],"foo","barval"); # cas success print $sock "cas foo 0 0 6 $result[0]\r\nbarva2\r\n"; is(scalar <$sock>, "STORED\r\n", "cas success, set foo"); # cas failure (reusing the same key) print $sock "cas foo 0 0 6 $result[0]\r\nbarva2\r\n"; is(scalar <$sock>, "EXISTS\r\n", "reusing a CAS ID"); # delete foo print $sock "delete foo\r\n"; is(scalar <$sock>, "DELETED\r\n", "deleted foo"); # cas missing print $sock "cas foo 0 0 6 $result[0]\r\nbarva2\r\n"; is(scalar <$sock>, "NOT_FOUND\r\n", "cas failed, foo does not exist"); # cas empty print $sock "cas foo 0 0 6 \r\nbarva2\r\n"; is(scalar <$sock>, "ERROR\r\n", "cas empty, throw error"); # cant parse barval2\r\n is(scalar <$sock>, "ERROR\r\n", "error out on barval2 parsing"); # set foo1 print $sock "set foo1 0 0 1\r\n1\r\n"; is(scalar <$sock>, "STORED\r\n", "set foo1"); # set foo2 print $sock "set foo2 0 0 1\r\n2\r\n"; is(scalar <$sock>, "STORED\r\n", "set foo2"); # gets foo1 check print $sock "gets foo1\r\n"; ok(scalar <$sock> =~ /VALUE foo1 0 1 (\d+)\r\n/, "gets foo1 regexp success"); my $foo1_cas = $1; is(scalar <$sock>, "1\r\n","gets foo1 data is 1"); is(scalar <$sock>, "END\r\n","gets foo1 END"); # gets foo2 check print $sock "gets foo2\r\n"; ok(scalar <$sock> =~ /VALUE foo2 0 1 (\d+)\r\n/,"gets foo2 regexp success"); my $foo2_cas = $1; is(scalar <$sock>, "2\r\n","gets foo2 data is 2"); is(scalar <$sock>, "END\r\n","gets foo2 END"); # validate foo1 != foo2 ok($foo1_cas != $foo2_cas,"foo1 != foo2 single-gets success"); # multi-gets print $sock "gets foo1 foo2\r\n"; ok(scalar <$sock> =~ /VALUE foo1 0 1 (\d+)\r\n/, "validating first set of data is foo1"); $foo1_cas = $1; is(scalar <$sock>, "1\r\n", "validating foo1 set of data is 1"); ok(scalar <$sock> =~ /VALUE foo2 0 1 (\d+)\r\n/, "validating second set of data is foo2"); $foo2_cas = $1; is(scalar <$sock>, "2\r\n", "validating foo2 set of data is 2"); is(scalar <$sock>, "END\r\n","validating foo1,foo2 gets is over - END"); # validate foo1 != foo2 ok($foo1_cas != $foo2_cas, "foo1 != foo2 multi-gets success"); ### simulate race condition with cas # gets foo1 - success @result = mem_gets($sock, "foo1"); ok($result[0] != "", "sock - gets foo1 is not empty"); # gets foo2 - success @result2 = mem_gets($sock2, "foo1"); ok($result2[0] != "","sock2 - gets foo1 is not empty"); print $sock "cas foo1 0 0 6 $result[0]\r\nbarva2\r\n"; print $sock2 "cas foo1 0 0 5 $result2[0]\r\napple\r\n"; my $res1 = <$sock>; my $res2 = <$sock2>; ok( ( $res1 eq "STORED\r\n" && $res2 eq "EXISTS\r\n") || ( $res1 eq "EXISTS\r\n" && $res2 eq "STORED\r\n"), "cas on same item from two sockets"); ### bug 15: http://code.google.com/p/memcached/issues/detail?id=15 # set foo print $sock "set bug15 0 0 1\r\n0\r\n"; is(scalar <$sock>, "STORED\r\n", "stored 0"); # Check out the first gets. print $sock "gets bug15\r\n"; ok(scalar <$sock> =~ /VALUE bug15 0 1 (\d+)\r\n/, "gets bug15 regexp success"); my $bug15_cas = $1; is(scalar <$sock>, "0\r\n", "gets bug15 data is 0"); is(scalar <$sock>, "END\r\n","gets bug15 END"); # Increment print $sock "incr bug15 1\r\n"; is(scalar <$sock>, "1\r\n", "incr worked"); # Validate a changed CAS print $sock "gets bug15\r\n"; ok(scalar <$sock> =~ /VALUE bug15 0 1 (\d+)\r\n/, "gets bug15 regexp success"); my $next_bug15_cas = $1; is(scalar <$sock>, "1\r\n", "gets bug15 data is 1"); is(scalar <$sock>, "END\r\n","gets bug15 END"); ok($bug15_cas != $next_bug15_cas, "CAS changed"); memcached-1.5.22/util.c0000664000175000017500000001464213611703161011561 00000000000000#include #include #include #include #include #include #include #include "memcached.h" static char *uriencode_map[256]; static char uriencode_str[768]; void uriencode_init(void) { int x; char *str = uriencode_str; for (x = 0; x < 256; x++) { if (isalnum(x) || x == '-' || x == '.' || x == '_' || x == '~') { uriencode_map[x] = NULL; } else { snprintf(str, 4, "%%%02hhX", (unsigned char)x); uriencode_map[x] = str; str += 3; /* lobbing off the \0 is fine */ } } } bool uriencode(const char *src, char *dst, const size_t srclen, const size_t dstlen) { int x; size_t d = 0; for (x = 0; x < srclen; x++) { if (d + 4 > dstlen) return false; if (uriencode_map[(unsigned char) src[x]] != NULL) { memcpy(&dst[d], uriencode_map[(unsigned char) src[x]], 3); d += 3; } else { dst[d] = src[x]; d++; } } dst[d] = '\0'; return true; } /* Avoid warnings on solaris, where isspace() is an index into an array, and gcc uses signed chars */ #define xisspace(c) isspace((unsigned char)c) bool safe_strtoull(const char *str, uint64_t *out) { assert(out != NULL); errno = 0; *out = 0; char *endptr; unsigned long long ull = strtoull(str, &endptr, 10); if ((errno == ERANGE) || (str == endptr)) { return false; } if (xisspace(*endptr) || (*endptr == '\0' && endptr != str)) { if ((long long) ull < 0) { /* only check for negative signs in the uncommon case when * the unsigned number is so big that it's negative as a * signed number. */ if (strchr(str, '-') != NULL) { return false; } } *out = ull; return true; } return false; } /* Could macro this. Decided to keep this unrolled for safety rather than add * the base parameter to all callers. Very few places need to parse a number * outside of base 10, currently exactly once, so splitting this up should * help avoid typo bugs. */ bool safe_strtoull_hex(const char *str, uint64_t *out) { assert(out != NULL); errno = 0; *out = 0; char *endptr; unsigned long long ull = strtoull(str, &endptr, 16); if ((errno == ERANGE) || (str == endptr)) { return false; } if (xisspace(*endptr) || (*endptr == '\0' && endptr != str)) { if ((long long) ull < 0) { /* only check for negative signs in the uncommon case when * the unsigned number is so big that it's negative as a * signed number. */ if (strchr(str, '-') != NULL) { return false; } } *out = ull; return true; } return false; } bool safe_strtoll(const char *str, int64_t *out) { assert(out != NULL); errno = 0; *out = 0; char *endptr; long long ll = strtoll(str, &endptr, 10); if ((errno == ERANGE) || (str == endptr)) { return false; } if (xisspace(*endptr) || (*endptr == '\0' && endptr != str)) { *out = ll; return true; } return false; } bool safe_strtoul(const char *str, uint32_t *out) { char *endptr = NULL; unsigned long l = 0; assert(out); assert(str); *out = 0; errno = 0; l = strtoul(str, &endptr, 10); if ((errno == ERANGE) || (str == endptr)) { return false; } if (xisspace(*endptr) || (*endptr == '\0' && endptr != str)) { if ((long) l < 0) { /* only check for negative signs in the uncommon case when * the unsigned number is so big that it's negative as a * signed number. */ if (strchr(str, '-') != NULL) { return false; } } *out = l; return true; } return false; } bool safe_strtol(const char *str, int32_t *out) { assert(out != NULL); errno = 0; *out = 0; char *endptr; long l = strtol(str, &endptr, 10); if ((errno == ERANGE) || (str == endptr)) { return false; } if (xisspace(*endptr) || (*endptr == '\0' && endptr != str)) { *out = l; return true; } return false; } bool safe_strtod(const char *str, double *out) { assert(out != NULL); errno = 0; *out = 0; char *endptr; double d = strtod(str, &endptr); if ((errno == ERANGE) || (str == endptr)) { return false; } if (xisspace(*endptr) || (*endptr == '\0' && endptr != str)) { *out = d; return true; } return false; } // slow, safe function for copying null terminated buffers. // ensures null terminator set on destination buffer. copies at most dstmax-1 // non-null bytes. // Explicitly avoids over-reading src while looking for the null byte. // returns true if src was fully copied. // returns false if src was truncated into dst. bool safe_strcpy(char *dst, const char *src, const size_t dstmax) { size_t x; for (x = 0; x < dstmax - 1 && src[x] != '\0'; x++) { dst[x] = src[x]; } dst[x] = '\0'; if (src[x] == '\0') { return true; } else { return false; } } bool safe_memcmp(const void *a, const void *b, size_t len) { const volatile unsigned char *ua = (const volatile unsigned char *)a; const volatile unsigned char *ub = (const volatile unsigned char *)b; int delta = 0; size_t x; for (x = 0; x < len; x++) { delta |= ua[x] ^ ub[x]; } if (delta == 0) { return true; } else { return false; } } void vperror(const char *fmt, ...) { int old_errno = errno; char buf[1024]; va_list ap; va_start(ap, fmt); if (vsnprintf(buf, sizeof(buf), fmt, ap) == -1) { buf[sizeof(buf) - 1] = '\0'; } va_end(ap); errno = old_errno; perror(buf); } #ifndef HAVE_HTONLL static uint64_t mc_swap64(uint64_t in) { #ifdef ENDIAN_LITTLE /* Little endian, flip the bytes around until someone makes a faster/better * way to do this. */ int64_t rv = 0; int i = 0; for(i = 0; i<8; i++) { rv = (rv << 8) | (in & 0xff); in >>= 8; } return rv; #else /* big-endian machines don't need byte swapping */ return in; #endif } uint64_t ntohll(uint64_t val) { return mc_swap64(val); } uint64_t htonll(uint64_t val) { return mc_swap64(val); } #endif memcached-1.5.22/itoa_ljust.c0000664000175000017500000001217213461170555012765 00000000000000//=== itoa_ljust.cpp - Fast integer to ascii conversion --*- C++ -*-// // // Substantially simplified (and slightly faster) version // based on the following functions in Google's protocol buffers: // // FastInt32ToBufferLeft() // FastUInt32ToBufferLeft() // FastInt64ToBufferLeft() // FastUInt64ToBufferLeft() // // Differences: // 1) Greatly simplified // 2) Avoids GOTO statements - uses "switch" instead and relies on // compiler constant folding and propagation for high performance // 3) Avoids unary minus of signed types - undefined behavior if value // is INT_MIN in platforms using two's complement representation // 4) Uses memcpy to store 2 digits at a time - lets the compiler // generate a 2-byte load/store in platforms that support // unaligned access, this is faster (and less code) than explicitly // loading and storing each byte // // Copyright (c) 2016 Arturo Martin-de-Nicolas // arturomdn@gmail.com // https://github.com/amdn/itoa_ljust/ // // Released under the BSD 3-Clause License, see Google's original copyright // and license below. //===----------------------------------------------------------------------===// // Protocol Buffers - Google's data interchange format // Copyright 2008 Google Inc. All rights reserved. // https://developers.google.com/protocol-buffers/ // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //===----------------------------------------------------------------------===// #include "itoa_ljust.h" #include static const char lut[201] = "0001020304050607080910111213141516171819" "2021222324252627282930313233343536373839" "4041424344454647484950515253545556575859" "6061626364656667686970717273747576777879" "8081828384858687888990919293949596979899"; #define dd(u) ((const uint16_t)(lut[u])) static inline char* out2(const int d, char* p) { memcpy(p, &((uint16_t *)lut)[d], 2); return p + 2; } static inline char* out1(const char in, char* p) { memcpy(p, &in, 1); return p + 1; } static inline int digits( uint32_t u, unsigned k, int* d, char** p, int n ) { if (u < k*10) { *d = u / k; *p = out1('0'+*d, *p); --n; } return n; } static inline char* itoa(uint32_t u, char* p, int d, int n) { switch(n) { case 10: d = u / 100000000; p = out2( d, p ); case 9: u -= d * 100000000; case 8: d = u / 1000000; p = out2( d, p ); case 7: u -= d * 1000000; case 6: d = u / 10000; p = out2( d, p ); case 5: u -= d * 10000; case 4: d = u / 100; p = out2( d, p ); case 3: u -= d * 100; case 2: d = u / 1; p = out2( d, p ); case 1: ; } *p = '\0'; return p; } char* itoa_u32(uint32_t u, char* p) { int d = 0,n; if (u >=100000000) n = digits(u, 100000000, &d, &p, 10); else if (u < 100) n = digits(u, 1, &d, &p, 2); else if (u < 10000) n = digits(u, 100, &d, &p, 4); else if (u < 1000000) n = digits(u, 10000, &d, &p, 6); else n = digits(u, 1000000, &d, &p, 8); return itoa( u, p, d, n ); } char* itoa_32(int32_t i, char* p) { uint32_t u = i; if (i < 0) { *p++ = '-'; u = -u; } return itoa_u32(u, p); } char* itoa_u64(uint64_t u, char* p) { int d; uint32_t lower = (uint32_t)u; if (lower == u) return itoa_u32(lower, p); uint64_t upper = u / 1000000000; p = itoa_u64(upper, p); lower = u - (upper * 1000000000); d = lower / 100000000; p = out1('0'+d,p); return itoa( lower, p, d, 9 ); } char* itoa_64(int64_t i, char* p) { uint64_t u = i; if (i < 0) { *p++ = '-'; u = -u; } return itoa_u64(u, p); } memcached-1.5.22/crawler.h0000664000175000017500000000245113575022500012243 00000000000000#ifndef CRAWLER_H #define CRAWLER_H #define LRU_CRAWLER_CAP_REMAINING -1 typedef struct { uint64_t histo[61]; uint64_t ttl_hourplus; uint64_t noexp; uint64_t reclaimed; uint64_t seen; rel_time_t start_time; rel_time_t end_time; bool run_complete; } crawlerstats_t; struct crawler_expired_data { pthread_mutex_t lock; crawlerstats_t crawlerstats[POWER_LARGEST]; /* redundant with crawlerstats_t so we can get overall start/stop/done */ rel_time_t start_time; rel_time_t end_time; bool crawl_complete; bool is_external; /* whether this was an alloc local or remote to the module. */ }; enum crawler_result_type { CRAWLER_OK=0, CRAWLER_RUNNING, CRAWLER_BADCLASS, CRAWLER_NOTSTARTED, CRAWLER_ERROR }; int start_item_crawler_thread(void); #define CRAWLER_WAIT true #define CRAWLER_NOWAIT false int stop_item_crawler_thread(bool wait); int init_lru_crawler(void *arg); enum crawler_result_type lru_crawler_crawl(char *slabs, enum crawler_run_type, void *c, const int sfd, unsigned int remaining); int lru_crawler_start(uint8_t *ids, uint32_t remaining, const enum crawler_run_type type, void *data, void *c, const int sfd); void lru_crawler_pause(void); void lru_crawler_resume(void); #endif memcached-1.5.22/memcached_dtrace.d0000664000175000017500000002472413575022500014037 00000000000000/* * Copyright (c) <2008>, Sun Microsystems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY SUN MICROSYSTEMS, INC. ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL SUN MICROSYSTEMS, INC. BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ provider memcached { /** * Fired when a connection object is allocated from the connection pool. * @param connid the connection id */ probe conn__allocate(int connid); /** * Fired when a connection object is released back to the connection pool. * @param connid the connection id */ probe conn__release(int connid); /** * Fired when a new connection object is created (there are no more * connection objects in the connection pool). * @param ptr pointer to the connection object */ probe conn__create(void *ptr); /** * Fired when a connection object is destroyed ("released back to * the memory subsystem"). * @param ptr pointer to the connection object */ probe conn__destroy(void *ptr); /** * Fired when a connection is dispatched from the "main thread" to a * worker thread. * @param connid the connection id * @param threadid the thread id */ probe conn__dispatch(int connid, int64_t threadid); /** * Allocate memory from the slab allocator. * @param size the requested size * @param slabclass the allocation will be fulfilled in this class * @param slabsize the size of each item in this class * @param ptr pointer to allocated memory */ probe slabs__allocate(int size, int slabclass, int slabsize, void* ptr); /** * Failed to allocate memory (out of memory). * @param size the requested size * @param slabclass the class that failed to fulfill the request */ probe slabs__allocate__failed(int size, int slabclass); /** * Fired when a slab class attempts to allocate more space. * @param slabclass class that needs more memory */ probe slabs__slabclass__allocate(int slabclass); /** * Failed to allocate memory (out of memory). * @param slabclass the class that failed grab more memory */ probe slabs__slabclass__allocate__failed(int slabclass); /** * Release memory. * @param size the size of the memory * @param slabclass the class the memory belongs to * @param ptr pointer to the memory to release */ probe slabs__free(int size, int slabclass, void* ptr); /** * Fired when the when we have searched the hash table for a named key. * These two elements provide an insight in how well the hash function * functions. Long traversals are a sign of a less optimal function, * wasting cpu capacity. * * @param key the key searched for * @param keylen length of the key * @param depth the depth in the list of hash table */ probe assoc__find(const char *key, int keylen, int depth); /** * Fired when a new item has been inserted. * @param key the key just inserted * @param keylen length of the key */ probe assoc__insert(const char *key, int keylen); /** * Fired when a new item has been removed. * @param key the key just deleted * @param keylen length of the key */ probe assoc__delete(const char *key, int keylen); /** * Fired when an item is linked into the cache. * @param key the items key * @param keylen length of the key * @param size the size of the data */ probe item__link(const char *key, int keylen, int size); /** * Fired when an item is deleted. * @param key the items key * @param keylen length of the key * @param size the size of the data */ probe item__unlink(const char *key, int keylen, int size); /** * Fired when the refcount for an item is reduced. * @param key the items key * @param keylen length of the key * @param size the size of the data */ probe item__remove(const char *key, int keylen, int size); /** * Fired when the "last refenced" time is updated. * @param key the items key * @param keylen length of the key * @param size the size of the data */ probe item__update(const char *key, int keylen, int size); /** * Fired when an item is replaced with another item. * @param oldkey the key of the item to replace * @param oldkeylen the length of the old key * @param oldsize the size of the old item * @param newkey the key of the new item * @param newkeylen the length of the new key * @param newsize the size of the new item */ probe item__replace(const char *oldkey, int oldkeylen, int oldsize, const char *newkey, int newkeylen, int newsize); /** * Fired when the processing of a command starts. * @param connid the connection id * @param request the incoming request * @param size the size of the request */ probe process__command__start(int connid, const void *request, int size); /** * Fired when the processing of a command is done. * @param connid the connection id * @param response the response to send back to the client * @param size the size of the response */ probe process__command__end(int connid, const void *response, int size); /** * Fired for a get-command * @param connid connection id * @param key requested key * @param keylen length of the key * @param size size of the key's data (or signed int -1 if not found) * @param casid the casid for the item */ probe command__get(int connid, const char *key, int keylen, int size, int64_t casid); /** * Fired for an add-command. * @param connid connection id * @param key requested key * @param keylen length of the key * @param size the new size of the key's data (or signed int -1 if * not found) * @param casid the casid for the item */ probe command__add(int connid, const char *key, int keylen, int size, int64_t casid); /** * Fired for a set-command. * @param connid connection id * @param key requested key * @param keylen length of the key * @param size the new size of the key's data (or signed int -1 if * not found) * @param casid the casid for the item */ probe command__set(int connid, const char *key, int keylen, int size, int64_t casid); /** * Fired for a replace-command. * @param connid connection id * @param key requested key * @param keylen length of the key * @param size the new size of the key's data (or signed int -1 if * not found) * @param casid the casid for the item */ probe command__replace(int connid, const char *key, int keylen, int size, int64_t casid); /** * Fired for a prepend-command. * @param connid connection id * @param key requested key * @param keylen length of the key * @param size the new size of the key's data (or signed int -1 if * not found) * @param casid the casid for the item */ probe command__prepend(int connid, const char *key, int keylen, int size, int64_t casid); /** * Fired for an append-command. * @param connid connection id * @param key requested key * @param keylen length of the key * @param size the new size of the key's data (or signed int -1 if * not found) * @param casid the casid for the item */ probe command__append(int connid, const char *key, int keylen, int size, int64_t casid); /** * Fired for an touch-command. * @param connid connection id * @param key requested key * @param keylen length of the key * @param size the new size of the key's data (or signed int -1 if * not found) * @param casid the casid for the item */ probe command__touch(int connid, const char *key, int keylen, int size, int64_t casid); /** * Fired for a cas-command. * @param connid connection id * @param key requested key * @param keylen length of the key * @param size size of the key's data (or signed int -1 if not found) * @param casid the cas id requested */ probe command__cas(int connid, const char *key, int keylen, int size, int64_t casid); /** * Fired for an incr command. * @param connid connection id * @param key the requested key * @param keylen length of the key * @param val the new value */ probe command__incr(int connid, const char *key, int keylen, int64_t val); /** * Fired for a decr command. * @param connid connection id * @param key the requested key * @param keylen length of the key * @param val the new value */ probe command__decr(int connid, const char *key, int keylen, int64_t val); /** * Fired for a delete command. * @param connid connection id * @param key the requested key * @param keylen length of the key */ probe command__delete(int connid, const char *key, int keylen); }; #pragma D attributes Unstable/Unstable/Common provider memcached provider #pragma D attributes Private/Private/Common provider memcached module #pragma D attributes Private/Private/Common provider memcached function #pragma D attributes Unstable/Unstable/Common provider memcached name #pragma D attributes Unstable/Unstable/Common provider memcached args memcached-1.5.22/slab_automove_extstore.h0000664000175000017500000000036613461170555015414 00000000000000#ifndef SLAB_AUTOMOVE_EXTSTORE_H #define SLAB_AUTOMOVE_EXTSTORE_H void *slab_automove_extstore_init(struct settings *settings); void slab_automove_extstore_free(void *arg); void slab_automove_extstore_run(void *arg, int *src, int *dst); #endif memcached-1.5.22/COPYING0000644000175000017500000000273711246331452011475 00000000000000Copyright (c) 2003, Danga Interactive, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the Danga Interactive nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. memcached-1.5.22/doc/0000755000175000017500000000000013615454046011265 500000000000000memcached-1.5.22/doc/protocol-binary.txt0000664000175000017500000017212013615454046015076 00000000000000 Network Working Group Stone, Ed. Internet-Draft Six Apart, Ltd. Intended status: Informational Norbye, Ed. Expires: March 1, 2009 Sun Microsystems, INC August 28, 2008 Memcache Binary Protocol draft-stone-memcache-binary-01 Abstract This memo explains the memcache binary protocol for informational purposes. Memcache is a high performance key-value cache. It is intentionally a dumb cache, optimized for speed only. Applications using memcache do not rely on it for data -- a persistent database with guaranteed reliability is strongly recommended -- but applications can run much faster when cached data is available in memcache. Status of This Memo This Internet-Draft is submitted in full conformance with the provisions of BCP 78 and BCP 79. Internet-Drafts are working documents of the Internet Engineering Task Force (IETF). Note that other groups may also distribute working documents as Internet-Drafts. The list of current Internet- Drafts is at http://datatracker.ietf.org/drafts/current/. Internet-Drafts are draft documents valid for a maximum of six months and may be updated, replaced, or obsoleted by other documents at any time. It is inappropriate to use Internet-Drafts as reference material or to cite them other than as "work in progress." This Internet-Draft will expire on March 1, 2009. Copyright Notice Copyright (c) 2008 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect Stone & Norbye Expires March 1, 2009 [Page 1] Internet-Draft Memcache Binary Protocol August 2008 to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 1.1. Conventions Used In This Document . . . . . . . . . . . . 3 2. Packet Structure . . . . . . . . . . . . . . . . . . . . . . 3 3. Defined Values . . . . . . . . . . . . . . . . . . . . . . . 5 3.1. Magic Byte . . . . . . . . . . . . . . . . . . . . . . . 5 3.2. Response Status . . . . . . . . . . . . . . . . . . . . . 5 3.3. Command Opcodes . . . . . . . . . . . . . . . . . . . . . 6 3.4. Data Types . . . . . . . . . . . . . . . . . . . . . . . 6 4. Commands . . . . . . . . . . . . . . . . . . . . . . . . . . 7 4.1. Introduction . . . . . . . . . . . . . . . . . . . . . . 7 4.1.1. Example . . . . . . . . . . . . . . . . . . . . . . . 7 4.2. Get, Get Quietly, Get Key, Get Key Quietly . . . . . . . 8 4.2.1. Example . . . . . . . . . . . . . . . . . . . . . . . 9 4.3. Set, Add, Replace . . . . . . . . . . . . . . . . . . . . 13 4.3.1. Example . . . . . . . . . . . . . . . . . . . . . . . 13 4.4. Delete . . . . . . . . . . . . . . . . . . . . . . . . . 15 4.4.1. Example . . . . . . . . . . . . . . . . . . . . . . . 16 4.5. Increment, Decrement . . . . . . . . . . . . . . . . . . 17 4.5.1. Example . . . . . . . . . . . . . . . . . . . . . . . 18 4.6. quit . . . . . . . . . . . . . . . . . . . . . . . . . . 20 4.6.1. Example . . . . . . . . . . . . . . . . . . . . . . . 21 4.7. Flush . . . . . . . . . . . . . . . . . . . . . . . . . . 21 4.7.1. Example . . . . . . . . . . . . . . . . . . . . . . . 22 4.8. noop . . . . . . . . . . . . . . . . . . . . . . . . . . 23 4.8.1. Example . . . . . . . . . . . . . . . . . . . . . . . 24 4.9. version . . . . . . . . . . . . . . . . . . . . . . . . . 24 4.9.1. Example . . . . . . . . . . . . . . . . . . . . . . . 25 4.10. Append, Prepend . . . . . . . . . . . . . . . . . . . . . 26 4.10.1. Example . . . . . . . . . . . . . . . . . . . . . . 27 4.11. Stat . . . . . . . . . . . . . . . . . . . . . . . . . . 28 4.11.1. Example . . . . . . . . . . . . . . . . . . . . . . 28 5. Security Considerations . . . . . . . . . . . . . . . . . . . 30 6. Normative References . . . . . . . . . . . . . . . . . . . . 31 Appendix A. Acknowledgments . . . . . . . . . . . . . . . . . . 31 Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . 31 1. Introduction Memcache is a high performance key-value cache. It is intentionally a dumb cache, optimized for speed only. Applications using memcache should not rely on it for data -- a persistent database with Stone & Norbye Expires March 1, 2009 [Page 2] Internet-Draft Memcache Binary Protocol August 2008 guaranteed reliability is strongly recommended -- but applications can run much faster when cached data is available in memcache. Memcache was originally written to make LiveJournal [LJ] faster. It now powers all of the fastest web sites that you love. 1.1. Conventions Used In This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [KEYWORDS]. 2. Packet Structure General format of a packet: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0/ HEADER / / / / / / / +---------------+---------------+---------------+---------------+ 24/ COMMAND-SPECIFIC EXTRAS (as needed) / +/ (note length in the extras length header field) / +---------------+---------------+---------------+---------------+ m/ Key (as needed) / +/ (note length in key length header field) / +---------------+---------------+---------------+---------------+ n/ Value (as needed) / +/ (note length is total body length header field, minus / +/ sum of the extras and key length body fields) / +---------------+---------------+---------------+---------------+ Total 24 bytes Stone & Norbye Expires March 1, 2009 [Page 3] Internet-Draft Memcache Binary Protocol August 2008 Request header: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Magic | Opcode | Key length | +---------------+---------------+---------------+---------------+ 4| Extras length | Data type | Reserved | +---------------+---------------+---------------+---------------+ 8| Total body length | +---------------+---------------+---------------+---------------+ 12| Opaque | +---------------+---------------+---------------+---------------+ 16| CAS | | | +---------------+---------------+---------------+---------------+ Total 24 bytes Response header: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Magic | Opcode | Key Length | +---------------+---------------+---------------+---------------+ 4| Extras length | Data type | Status | +---------------+---------------+---------------+---------------+ 8| Total body length | +---------------+---------------+---------------+---------------+ 12| Opaque | +---------------+---------------+---------------+---------------+ 16| CAS | | | +---------------+---------------+---------------+---------------+ Total 24 bytes Header fields: Magic Magic number. Opcode Command code. Key length Length in bytes of the text key that follows the command extras. Status Status of the response (non-zero on error). Extras length Length in bytes of the command extras. Data type Reserved for future use (Sean is using this soon). Stone & Norbye Expires March 1, 2009 [Page 4] Internet-Draft Memcache Binary Protocol August 2008 Reserved Really reserved for future use (up for grabs). Total body length Length in bytes of extra + key + value. Opaque Will be copied back to you in the response. CAS Data version check. 3. Defined Values 3.1. Magic Byte 0x80 Request packet for this protocol version 0x81 Response packet for this protocol version Magic byte / version. For each version of the protocol, we'll use a different request/response value pair. This is useful for protocol analyzers to distinguish the nature of the packet from the direction which it is moving. Note, it is common to run a memcached instance on a host that also runs an application server. Such a host will both send and receive memcache packets. The version should hopefully correspond only to different meanings of the command byte. In an ideal world, we will not change the header format. As reserved bytes are given defined meaning, the protocol version / magic byte values should be incremented. Traffic analysis tools are encouraged to identify memcache packets and provide detailed interpretation if the magic bytes are recognized and otherwise to provide a generic breakdown of the packet. Note, that the key and value positions can always be identified even if the magic byte or command opcode are not recognized. 3.2. Response Status Possible values of this two-byte field: 0x0000 No error 0x0001 Key not found 0x0002 Key exists 0x0003 Value too large 0x0004 Invalid arguments 0x0005 Item not stored 0x0006 Incr/Decr on non-numeric value. 0x0081 Unknown command 0x0082 Out of memory Stone & Norbye Expires March 1, 2009 [Page 5] Internet-Draft Memcache Binary Protocol August 2008 3.3. Command Opcodes Possible values of the one-byte field: 0x00 Get 0x01 Set 0x02 Add 0x03 Replace 0x04 Delete 0x05 Increment 0x06 Decrement 0x07 Quit 0x08 Flush 0x09 GetQ 0x0A No-op 0x0B Version 0x0C GetK 0x0D GetKQ 0x0E Append 0x0F Prepend 0x10 Stat 0x11 SetQ 0x12 AddQ 0x13 ReplaceQ 0x14 DeleteQ 0x15 IncrementQ 0x16 DecrementQ 0x17 QuitQ 0x18 FlushQ 0x19 AppendQ 0x1A PrependQ As a convention all of the commands ending with "Q" for Quiet. A quiet version of a command will omit responses that are considered uninteresting. Whether a given response is interesting is dependent upon the command. See the descriptions of the set commands (Section 4.2) and set commands (Section 4.3) for examples of commands that include quiet variants. 3.4. Data Types Possible values of the one-byte field: 0x00 Raw bytes Stone & Norbye Expires March 1, 2009 [Page 6] Internet-Draft Memcache Binary Protocol August 2008 4. Commands 4.1. Introduction All communication is initiated by a request from the client, and the server will respond to each request with zero or multiple packets for each request. If the status code of a response packet is non-nil, the body of the packet will contain a textual error message. If the status code is nil, the command opcode will define the layout of the body of the message. 4.1.1. Example The following figure illustrates the packet layout for a packet with an error message. Stone & Norbye Expires March 1, 2009 [Page 7] Internet-Draft Memcache Binary Protocol August 2008 Packet layout: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x09 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x4e ('N') | 0x6f ('o') | 0x74 ('t') | 0x20 (' ') | +---------------+---------------+---------------+---------------+ 28| 0x66 ('f') | 0x6f ('o') | 0x75 ('u') | 0x6e ('n') | +---------------+---------------+---------------+---------------+ 32| 0x64 ('d') | +---------------+ Total 33 bytes (24 byte header, and 9 bytes value) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x00 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0001 Total body (8-11) : 0x00000009 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value (24-32): The textual string "Not found" 4.2. Get, Get Quietly, Get Key, Get Key Quietly Request: MUST NOT have extras. MUST have key. MUST NOT have value. Stone & Norbye Expires March 1, 2009 [Page 8] Internet-Draft Memcache Binary Protocol August 2008 Response (if found): MUST have extras. MAY have key. MAY have value. o 4 byte flags Extra data for the get commands: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Flags | +---------------+---------------+---------------+---------------+ Total 4 bytes The get command gets a single key. The getq command is both mum on cache miss and quiet, holding its response until a non-quiet command is issued. Getk and getkq differs from get and getq by adding the key into the response packet. You're not guaranteed a response to a getq/getkq cache hit until you send a non-getq/getkq command later, which uncorks the server and bundles up IOs to send to the client in one go. Clients should implement multi-get (still important for reducing network roundtrips!) as n pipelined requests, the first n-1 being getq/getkq, the last being a regular get/getk. That way you're guaranteed to get a response, and you know when the server's done. You can also do the naive thing and send n pipelined get/getks, but then you could potentially get back a lot of "NOT_FOUND" error code packets. Alternatively, you can send 'n' getq/getkqs, followed by a 'noop' command. 4.2.1. Example To request the data associated with the key "Hello" the following fields must be specified in the packet. Stone & Norbye Expires March 1, 2009 [Page 9] Internet-Draft Memcache Binary Protocol August 2008 get request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 28| 0x6f ('o') | +---------------+ Total 29 bytes (24 byte header, and 5 bytes key) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x00 Key length (2,3) : 0x0005 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000005 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key (24-29): The textual string: "Hello" Value : None If the item exist on the server the following packet is returned, otherwise a packet with status code != 0 will be returned (see Introduction (Section 4.1)) Stone & Norbye Expires March 1, 2009 [Page 10] Internet-Draft Memcache Binary Protocol August 2008 get/getq response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x04 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x09 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ 24| 0xde | 0xad | 0xbe | 0xef | +---------------+---------------+---------------+---------------+ 28| 0x57 ('W') | 0x6f ('o') | 0x72 ('r') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 32| 0x64 ('d') | +---------------+ Total 33 bytes (24 byte header, 4 byte extras and 5 byte value) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x00 Key length (2,3) : 0x0000 Extra length (4) : 0x04 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000009 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000001 Extras : Flags (24-27): 0xdeadbeef Key : None Value (28-32): The textual string "World" Stone & Norbye Expires March 1, 2009 [Page 11] Internet-Draft Memcache Binary Protocol August 2008 getk/getkq response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x04 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x09 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ 24| 0xde | 0xad | 0xbe | 0xef | +---------------+---------------+---------------+---------------+ 28| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 32| 0x6f ('o') | 0x57 ('W') | 0x6f ('o') | 0x72 ('r') | +---------------+---------------+---------------+---------------+ 36| 0x6c ('l') | 0x64 ('d') | +---------------+---------------+ Total 38 bytes (24 byte header, 4 byte extras, 5 byte key and 5 byte value) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x00 Key length (2,3) : 0x0005 Extra length (4) : 0x04 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x0000000E Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000001 Extras : Flags (24-27): 0xdeadbeef Key (28-32): The textual string: "Hello" Value (33-37): The textual string: "World" Stone & Norbye Expires March 1, 2009 [Page 12] Internet-Draft Memcache Binary Protocol August 2008 4.3. Set, Add, Replace MUST have extras. MUST have key. MUST have value. o 4 byte flags o 4 byte expiration time Extra data for set/add/replace: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Flags | +---------------+---------------+---------------+---------------+ 4| Expiration | +---------------+---------------+---------------+---------------+ Total 8 bytes If the Data Version Check (CAS) is nonzero, the requested operation MUST only succeed if the item exists and has a CAS value identical to the provided value. Add MUST fail if the item already exist. Replace MUST fail if the item doesn't exist. Set should store the data unconditionally if the item exists or not. Quiet mutations only return responses on failure. Success is considered the general case and is suppressed when in quiet mode, but errors should not be allowed to go unnoticed. 4.3.1. Example The following figure shows an add-command for Key: "Hello" Value: "World" Flags: 0xdeadbeef Expiry: in two hours Stone & Norbye Expires March 1, 2009 [Page 13] Internet-Draft Memcache Binary Protocol August 2008 Add request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x02 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x08 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x12 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0xde | 0xad | 0xbe | 0xef | +---------------+---------------+---------------+---------------+ 28| 0x00 | 0x00 | 0x1c | 0x20 | +---------------+---------------+---------------+---------------+ 32| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 36| 0x6f ('o') | 0x57 ('W') | 0x6f ('o') | 0x72 ('r') | +---------------+---------------+---------------+---------------+ 40| 0x6c ('l') | 0x64 ('d') | +---------------+---------------+ Total 42 bytes (24 byte header, 8 byte extras, 5 byte key and 5 byte value) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x02 Key length (2,3) : 0x0005 Extra length (4) : 0x08 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000012 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : Flags (24-27): 0xdeadbeef Expiry (28-31): 0x00001c20 Key (32-36): The textual string "Hello" Value (37-41): The textual string "World" Stone & Norbye Expires March 1, 2009 [Page 14] Internet-Draft Memcache Binary Protocol August 2008 The response-packet contains no extra data, and the result of the operation is signaled through the status code. If the command succeeds, the CAS value for the item is returned in the CAS-field of the packet. Successful add response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x02 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x02 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000001 Extras : None Key : None Value : None 4.4. Delete MUST NOT have extras. MUST have key. MUST NOT have value. Delete the item with the specific key. Stone & Norbye Expires March 1, 2009 [Page 15] Internet-Draft Memcache Binary Protocol August 2008 4.4.1. Example The following figure shows a delete message for the item "Hello". Delete request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x04 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 28| 0x6f ('o') | +---------------+ Total 29 bytes (24 byte header, 5 byte value) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x04 Key length (2,3) : 0x0005 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000005 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : The textual string "Hello" Value : None The response-packet contains no extra data, and the result of the operation is signaled through the status code. Stone & Norbye Expires March 1, 2009 [Page 16] Internet-Draft Memcache Binary Protocol August 2008 4.5. Increment, Decrement MUST have extras. MUST have key. MUST NOT have value. o 8 byte value to add / subtract o 8 byte initial value (unsigned) o 4 byte expiration time Extra data for incr/decr: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Amount to add | | | +---------------+---------------+---------------+---------------+ 8| Initial value | | | +---------------+---------------+---------------+---------------+ 16| Expiration | +---------------+---------------+---------------+---------------+ Total 20 bytes These commands will either add or remove the specified amount to the requested counter. If the counter does not exist, one of two things may happen: 1. If the expiration value is all one-bits (0xffffffff), the operation will fail with NOT_FOUND. 2. For all other expiration values, the operation will succeed by seeding the value for this key with the provided initial value to expire with the provided expiration time. The flags will be set to zero. incr/decr response body: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 64-bit unsigned response. | | | +---------------+---------------+---------------+---------------+ Total 8 bytes Stone & Norbye Expires March 1, 2009 [Page 17] Internet-Draft Memcache Binary Protocol August 2008 4.5.1. Example The following figure shows an incr-command for Key: "counter" Delta: 0x01 Initial: 0x00 Expiry: in two hours Increment request: Stone & Norbye Expires March 1, 2009 [Page 18] Internet-Draft Memcache Binary Protocol August 2008 Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x05 | 0x00 | 0x07 | +---------------+---------------+---------------+---------------+ 4| 0x14 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x1b | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 28| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ 32| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 36| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 40| 0x00 | 0x00 | 0x1c | 0x20 | +---------------+---------------+---------------+---------------+ 44| 0x63 ('c') | 0x6f ('o') | 0x75 ('u') | 0x6e ('n') | +---------------+---------------+---------------+---------------+ 48| 0x74 ('t') | 0x65 ('e') | 0x72 ('r') | +---------------+---------------+---------------+ Total 51 bytes (24 byte header, 20 byte extras, 7 byte key) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x05 Key length (2,3) : 0x0007 Extra length (4) : 0x14 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x0000001b Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : delta (24-31): 0x0000000000000001 initial (32-39): 0x0000000000000000 expiration (40-43): 0x00001c20 Key : Textual string "counter" Value : None Stone & Norbye Expires March 1, 2009 [Page 19] Internet-Draft Memcache Binary Protocol August 2008 If the key doesn't exist, the server will respond with the initial value. If not the incremented value will be returned. Let's assume that the key didn't exist, so the initial value is returned. Increment response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x05 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x08 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 24| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 28| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 32 bytes (24 byte header, 8 byte value) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x05 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000008 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000005 Extras : None Key : None Value : 0x0000000000000000 4.6. quit MUST NOT have extras. MUST NOT have key. MUST NOT have value. Stone & Norbye Expires March 1, 2009 [Page 20] Internet-Draft Memcache Binary Protocol August 2008 Close the connection to the server. 4.6.1. Example Quit request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x07 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x07 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value : None The response-packet contains no extra data, and the result of the operation is signaled through the status code. The server will then close the connection. 4.7. Flush MAY have extras. MUST NOT have key. MUST NOT have value. Stone & Norbye Expires March 1, 2009 [Page 21] Internet-Draft Memcache Binary Protocol August 2008 o 4 byte expiration time Extra data for flush: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Expiration | +---------------+---------------+---------------+---------------+ Total 4 bytes Flush the items in the cache now or some time in the future as specified by the expiration field. See the documentation of the textual protocol for the full description on how to specify the expiration time. 4.7.1. Example To flush the cache (delete all items) in two hours, the set the following values in the request Stone & Norbye Expires March 1, 2009 [Page 22] Internet-Draft Memcache Binary Protocol August 2008 Flush request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x08 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x04 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x04 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x00 | 0x00 | 0x1c | 0x20 | +---------------+---------------+---------------+---------------+ Total 28 bytes (24 byte header, 4 byte body) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x08 Key length (2,3) : 0x0000 Extra length (4) : 0x04 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000004 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : Expiry (24-27): 0x00001c20 Key : None Value : None The response-packet contains no extra data, and the result of the operation is signaled through the status code. 4.8. noop MUST NOT have extras. MUST NOT have key. MUST NOT have value. Used as a keep alive. Flushes outstanding getq/getkq's. Stone & Norbye Expires March 1, 2009 [Page 23] Internet-Draft Memcache Binary Protocol August 2008 4.8.1. Example Noop request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x0a | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x0a Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value : None The response-packet contains no extra data, and the result of the operation is signaled through the status code. 4.9. version MUST NOT have extras. MUST NOT have key. MUST NOT have value. Request the server version. Stone & Norbye Expires March 1, 2009 [Page 24] Internet-Draft Memcache Binary Protocol August 2008 The server responds with a packet containing the version string in the body with the following format: "x.y.z" 4.9.1. Example Version request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x0b | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x0b Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Stone & Norbye Expires March 1, 2009 [Page 25] Internet-Draft Memcache Binary Protocol August 2008 Version response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x0b | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x31 ('1') | 0x2e ('.') | 0x33 ('3') | 0x2e ('.') | +---------------+---------------+---------------+---------------+ 28| 0x31 ('1') | +---------------+ Total 29 bytes (24 byte header, 5 byte body) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x0b Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000005 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value : Textual string "1.3.1" 4.10. Append, Prepend MUST NOT have extras. MUST have key. MUST have value. These commands will either append or prepend the specified value to the requested key. Stone & Norbye Expires March 1, 2009 [Page 26] Internet-Draft Memcache Binary Protocol August 2008 4.10.1. Example The following example appends '!' to the 'Hello' key. Append request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x0e | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x06 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 28| 0x6f ('o') | 0x21 ('!') | +---------------+---------------+ Total 30 bytes (24 byte header, 5 byte key, 1 byte value) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x0e Key length (2,3) : 0x0005 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000006 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key (24-28): The textual string "Hello" Value (29) : "!" The response-packet contains no extra data, and the result of the operation is signaled through the status code. Stone & Norbye Expires March 1, 2009 [Page 27] Internet-Draft Memcache Binary Protocol August 2008 4.11. Stat MUST NOT have extras. MAY have key. MUST NOT have value. Request server statistics. Without a key specified the server will respond with a "default" set of statistics information. Each piece of statistical information is returned in its own packet (key contains the name of the statistical item and the body contains the value in ASCII format). The sequence of return packets is terminated with a packet that contains no key and no value. 4.11.1. Example The following example requests all statistics from the server Stone & Norbye Expires March 1, 2009 [Page 28] Internet-Draft Memcache Binary Protocol August 2008 Stat request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x10 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x10 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value : None The server will send each value in a separate packet with an "empty" packet (no key / no value) to terminate the sequence. Each of the response packets look like the following example: Stone & Norbye Expires March 1, 2009 [Page 29] Internet-Draft Memcache Binary Protocol August 2008 Stat response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x10 | 0x00 | 0x03 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x07 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x70 ('p') | 0x69 ('i') | 0x64 ('d') | 0x33 ('3') | +---------------+---------------+---------------+---------------+ 28| 0x30 ('0') | 0x37 ('7') | 0x38 ('8') | +---------------+---------------+---------------+ Total 31 bytes (24 byte header, 3 byte key, 4 byte body) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x10 Key length (2,3) : 0x0003 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000007 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : The textual string "pid" Value : The textual string "3078" 5. Security Considerations Memcache has few authentication and no security layers whatsoever. It is RECOMMENDED that memcache be deployed strictly on closed, protected, back-end networks within a single data center, within a single cluster of servers, or even on a single host, providing shared caching for multiple applications. Memcache MUST NOT be made available on a public network. Stone & Norbye Expires March 1, 2009 [Page 30] Internet-Draft Memcache Binary Protocol August 2008 SASL is supported as an authentication mechanism. See the wiki for more information. 6. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [LJ] Danga Interactive, "LJ NEEDS MOAR SPEED", 10 1999. Appendix A. Acknowledgments Thanks to Brad Fitzpatrick, Anatoly Vorobey, Steven Grimm, and Dustin Sallings, for their work on the memcached server. Thanks to Sean Chittenden, Jonathan Steinert, Brian Aker, Evan Martin, Nathan Neulinger, Eric Hodel, Michael Johnson, Paul Querna, Jamie McCarthy, Philip Neustrom, Andrew O'Brien, Josh Rotenberg, Robin H. Johnson, Tim Yardley, Paolo Borelli, Eli Bingham, Jean- Francois Bustarret, Paul G, Paul Lindner, Alan Kasindorf, Chris Goffinet, Tomash Brechko, and others for their work reporting bugs and maintaining memcached client libraries and bindings in many languages. Authors' Addresses Aaron Stone (editor) Six Apart, Ltd. 548 4th Street San Francisco, CA 94107 USA Email: aaron@serendipity.palo-alto.ca.us Trond Norbye (editor) Sun Microsystems, INC Haakon VII g. 7B Trondheim NO-7485 Trondheim Norway Email: trond.norbye@sun.com Stone & Norbye Expires March 1, 2009 [Page 31] memcached-1.5.22/doc/xml2rfc/0000775000175000017500000000000013575022500012633 500000000000000memcached-1.5.22/doc/xml2rfc/reference.RFC.0768.xml0000664000175000017500000000130013461170555016251 00000000000000 User Datagram Protocol University of Southern California (USC)/Information Sciences Institute
4676 Admiralty Way Marina del Rey CA 90291 US +1 213 822 1511
 memcached-1.5.22/doc/xml2rfc/rfc2629-xhtml.ent0000664000175000017500000003617013461170555015531 00000000000000 memcached-1.5.22/doc/xml2rfc/reference.RFC.2119.xml0000664000175000017500000000324013461170555016246 00000000000000 Key words for use in RFCs to Indicate Requirement Levels Harvard University
1350 Mass. Ave. Cambridge MA 02138 - +1 617 495 3864 sob@harvard.edu
 General keyword In many standards track documents several words are used to signify the requirements in the specification. These words are often capitalized. This document defines these words as they should be interpreted in IETF documents. Authors who follow these guidelines should incorporate this phrase near the beginning of their document: The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119. Note that the force of these words is modified by the requirement level of the document in which they are used. memcached-1.5.22/doc/xml2rfc/rfc2629-noinc.xsl0000664000175000017500000000601213461170555015513 00000000000000 .xml memcached-1.5.22/doc/xml2rfc/rfc2629.dtd0000664000175000017500000002211013575022500014341 00000000000000 %rfc2629-xhtml; %rfc2629-other; memcached-1.5.22/doc/xml2rfc/rfc2629-refchk.xsl0000664000175000017500000000500613461170555015651 00000000000000 memcached-1.5.22/doc/xml2rfc/rfc2629-other.ent0000664000175000017500000000725413461170555015517 00000000000000 memcached-1.5.22/doc/protocol-binary.xml0000664000175000017500000017260213461170555015063 00000000000000 Memcache Binary Protocol Six Apart, Ltd.
548 4th Street San Francisco CA 94107 USA aaron@serendipity.palo-alto.ca.us
Sun Microsystems, INC
Haakon VII g. 7B Trondheim NO-7485 Trondheim Norway trond.norbye@sun.com
Applications memcache memcached cache This memo explains the memcache binary protocol for informational purposes. Memcache is a high performance key-value cache. It is intentionally a dumb cache, optimized for speed only. Applications using memcache do not rely on it for data -- a persistent database with guaranteed reliability is strongly recommended -- but applications can run much faster when cached data is available in memcache.
Memcache is a high performance key-value cache. It is intentionally a dumb cache, optimized for speed only. Applications using memcache should not rely on it for data -- a persistent database with guaranteed reliability is strongly recommended -- but applications can run much faster when cached data is available in memcache. Memcache was originally written to make LiveJournal faster. It now powers all of the fastest web sites that you love.
The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in .
General format of a packet: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0/ HEADER / / / / / / / +---------------+---------------+---------------+---------------+ 24/ COMMAND-SPECIFIC EXTRAS (as needed) / +/ (note length in the extras length header field) / +---------------+---------------+---------------+---------------+ m/ Key (as needed) / +/ (note length in key length header field) / +---------------+---------------+---------------+---------------+ n/ Value (as needed) / +/ (note length is total body length header field, minus / +/ sum of the extras and key length body fields) / +---------------+---------------+---------------+---------------+ Total 24 bytes
Request header: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Magic | Opcode | Key length | +---------------+---------------+---------------+---------------+ 4| Extras length | Data type | Reserved | +---------------+---------------+---------------+---------------+ 8| Total body length | +---------------+---------------+---------------+---------------+ 12| Opaque | +---------------+---------------+---------------+---------------+ 16| CAS | | | +---------------+---------------+---------------+---------------+ Total 24 bytes
Response header: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Magic | Opcode | Key Length | +---------------+---------------+---------------+---------------+ 4| Extras length | Data type | Status | +---------------+---------------+---------------+---------------+ 8| Total body length | +---------------+---------------+---------------+---------------+ 12| Opaque | +---------------+---------------+---------------+---------------+ 16| CAS | | | +---------------+---------------+---------------+---------------+ Total 24 bytes
Header fields: Magic number. Command code. Length in bytes of the text key that follows the command extras. Status of the response (non-zero on error). Length in bytes of the command extras. Reserved for future use (Sean is using this soon). Really reserved for future use (up for grabs). Length in bytes of extra + key + value. Will be copied back to you in the response. Data version check.
Request packet for this protocol version Response packet for this protocol version Magic byte / version. For each version of the protocol, we'll use a different request/response value pair. This is useful for protocol analyzers to distinguish the nature of the packet from the direction which it is moving. Note, it is common to run a memcached instance on a host that also runs an application server. Such a host will both send and receive memcache packets. The version should hopefully correspond only to different meanings of the command byte. In an ideal world, we will not change the header format. As reserved bytes are given defined meaning, the protocol version / magic byte values should be incremented. Traffic analysis tools are encouraged to identify memcache packets and provide detailed interpretation if the magic bytes are recognized and otherwise to provide a generic breakdown of the packet. Note, that the key and value positions can always be identified even if the magic byte or command opcode are not recognized.
Possible values of this two-byte field: No error Key not found Key exists Value too large Invalid arguments Item not stored Incr/Decr on non-numeric value. Unknown command Out of memory
Possible values of the one-byte field: Get Set Add Replace Delete Increment Decrement Quit Flush GetQ No-op Version GetK GetKQ Append Prepend Stat SetQ AddQ ReplaceQ DeleteQ IncrementQ DecrementQ QuitQ FlushQ AppendQ PrependQ As a convention all of the commands ending with "Q" for Quiet. A quiet version of a command will omit responses that are considered uninteresting. Whether a given response is interesting is dependent upon the command. See the descriptions of the set commands and set commands for examples of commands that include quiet variants.
Possible values of the one-byte field: Raw bytes
All communication is initiated by a request from the client, and the server will respond to each request with zero or multiple packets for each request. If the status code of a response packet is non-nil, the body of the packet will contain a textual error message. If the status code is nil, the command opcode will define the layout of the body of the message.
The following figure illustrates the packet layout for a packet with an error message.
Packet layout: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x09 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x4e ('N') | 0x6f ('o') | 0x74 ('t') | 0x20 (' ') | +---------------+---------------+---------------+---------------+ 28| 0x66 ('f') | 0x6f ('o') | 0x75 ('u') | 0x6e ('n') | +---------------+---------------+---------------+---------------+ 32| 0x64 ('d') | +---------------+ Total 33 bytes (24 byte header, and 9 bytes value) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x00 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0001 Total body (8-11) : 0x00000009 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value (24-32): The textual string "Not found"
Request: MUST NOT have extras. MUST have key. MUST NOT have value. Response (if found): MUST have extras. MAY have key. MAY have value. 4 byte flags
Extra data for the get commands: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Flags | +---------------+---------------+---------------+---------------+ Total 4 bytes
The get command gets a single key. The getq command is both mum on cache miss and quiet, holding its response until a non-quiet command is issued. Getk and getkq differs from get and getq by adding the key into the response packet. You're not guaranteed a response to a getq/getkq cache hit until you send a non-getq/getkq command later, which uncorks the server and bundles up IOs to send to the client in one go. Clients should implement multi-get (still important for reducing network roundtrips!) as n pipelined requests, the first n-1 being getq/getkq, the last being a regular get/getk. That way you're guaranteed to get a response, and you know when the server's done. You can also do the naive thing and send n pipelined get/getks, but then you could potentially get back a lot of "NOT_FOUND" error code packets. Alternatively, you can send 'n' getq/getkqs, followed by a 'noop' command.
To request the data associated with the key "Hello" the following fields must be specified in the packet.
get request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 28| 0x6f ('o') | +---------------+ Total 29 bytes (24 byte header, and 5 bytes key) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x00 Key length (2,3) : 0x0005 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000005 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key (24-29): The textual string: "Hello" Value : None
If the item exist on the server the following packet is returned, otherwise a packet with status code != 0 will be returned (see Introduction)
get/getq response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x04 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x09 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ 24| 0xde | 0xad | 0xbe | 0xef | +---------------+---------------+---------------+---------------+ 28| 0x57 ('W') | 0x6f ('o') | 0x72 ('r') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 32| 0x64 ('d') | +---------------+ Total 33 bytes (24 byte header, 4 byte extras and 5 byte value) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x00 Key length (2,3) : 0x0000 Extra length (4) : 0x04 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000009 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000001 Extras : Flags (24-27): 0xdeadbeef Key : None Value (28-32): The textual string "World"
getk/getkq response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x04 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x09 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ 24| 0xde | 0xad | 0xbe | 0xef | +---------------+---------------+---------------+---------------+ 28| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 32| 0x6f ('o') | 0x57 ('W') | 0x6f ('o') | 0x72 ('r') | +---------------+---------------+---------------+---------------+ 36| 0x6c ('l') | 0x64 ('d') | +---------------+---------------+ Total 38 bytes (24 byte header, 4 byte extras, 5 byte key and 5 byte value) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x00 Key length (2,3) : 0x0005 Extra length (4) : 0x04 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x0000000E Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000001 Extras : Flags (24-27): 0xdeadbeef Key (28-32): The textual string: "Hello" Value (33-37): The textual string: "World"
MUST have extras. MUST have key. MUST have value. 4 byte flags 4 byte expiration time
Extra data for set/add/replace: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Flags | +---------------+---------------+---------------+---------------+ 4| Expiration | +---------------+---------------+---------------+---------------+ Total 8 bytes
If the Data Version Check (CAS) is nonzero, the requested operation MUST only succeed if the item exists and has a CAS value identical to the provided value. Add MUST fail if the item already exist. Replace MUST fail if the item doesn't exist. Set should store the data unconditionally if the item exists or not. Quiet mutations only return responses on failure. Success is considered the general case and is suppressed when in quiet mode, but errors should not be allowed to go unnoticed.
The following figure shows an add-command for Key: "Hello" Value: "World" Flags: 0xdeadbeef Expiry: in two hours
Add request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x02 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x08 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x12 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0xde | 0xad | 0xbe | 0xef | +---------------+---------------+---------------+---------------+ 28| 0x00 | 0x00 | 0x1c | 0x20 | +---------------+---------------+---------------+---------------+ 32| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 36| 0x6f ('o') | 0x57 ('W') | 0x6f ('o') | 0x72 ('r') | +---------------+---------------+---------------+---------------+ 40| 0x6c ('l') | 0x64 ('d') | +---------------+---------------+ Total 42 bytes (24 byte header, 8 byte extras, 5 byte key and 5 byte value) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x02 Key length (2,3) : 0x0005 Extra length (4) : 0x08 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000012 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : Flags (24-27): 0xdeadbeef Expiry (28-31): 0x00001c20 Key (32-36): The textual string "Hello" Value (37-41): The textual string "World"
The response-packet contains no extra data, and the result of the operation is signaled through the status code. If the command succeeds, the CAS value for the item is returned in the CAS-field of the packet.
Successful add response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x02 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x02 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000001 Extras : None Key : None Value : None
MUST NOT have extras. MUST have key. MUST NOT have value. Delete the item with the specific key.
The following figure shows a delete message for the item "Hello".
Delete request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x04 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 28| 0x6f ('o') | +---------------+ Total 29 bytes (24 byte header, 5 byte value) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x04 Key length (2,3) : 0x0005 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000005 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : The textual string "Hello" Value : None
The response-packet contains no extra data, and the result of the operation is signaled through the status code.
MUST have extras. MUST have key. MUST NOT have value. 8 byte value to add / subtract 8 byte initial value (unsigned) 4 byte expiration time
Extra data for incr/decr: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Amount to add | | | +---------------+---------------+---------------+---------------+ 8| Initial value | | | +---------------+---------------+---------------+---------------+ 16| Expiration | +---------------+---------------+---------------+---------------+ Total 20 bytes
These commands will either add or remove the specified amount to the requested counter. If the counter does not exist, one of two things may happen: If the expiration value is all one-bits (0xffffffff), the operation will fail with NOT_FOUND. For all other expiration values, the operation will succeed by seeding the value for this key with the provided initial value to expire with the provided expiration time. The flags will be set to zero.
incr/decr response body: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 64-bit unsigned response. | | | +---------------+---------------+---------------+---------------+ Total 8 bytes
The following figure shows an incr-command for Key: "counter" Delta: 0x01 Initial: 0x00 Expiry: in two hours
Increment request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x05 | 0x00 | 0x07 | +---------------+---------------+---------------+---------------+ 4| 0x14 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x1b | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 28| 0x00 | 0x00 | 0x00 | 0x01 | +---------------+---------------+---------------+---------------+ 32| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 36| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 40| 0x00 | 0x00 | 0x1c | 0x20 | +---------------+---------------+---------------+---------------+ 44| 0x63 ('c') | 0x6f ('o') | 0x75 ('u') | 0x6e ('n') | +---------------+---------------+---------------+---------------+ 48| 0x74 ('t') | 0x65 ('e') | 0x72 ('r') | +---------------+---------------+---------------+ Total 51 bytes (24 byte header, 20 byte extras, 7 byte key) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x05 Key length (2,3) : 0x0007 Extra length (4) : 0x14 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x0000001b Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : delta (24-31): 0x0000000000000001 initial (32-39): 0x0000000000000000 expiration (40-43): 0x00001c20 Key : Textual string "counter" Value : None
If the key doesn't exist, the server will respond with the initial value. If not the incremented value will be returned. Let's assume that the key didn't exist, so the initial value is returned.
Increment response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x05 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x08 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 24| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 28| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 32 bytes (24 byte header, 8 byte value) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x05 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000008 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000005 Extras : None Key : None Value : 0x0000000000000000
MUST NOT have extras. MUST NOT have key. MUST NOT have value. Close the connection to the server.
Quit request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x07 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x07 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value : None
The response-packet contains no extra data, and the result of the operation is signaled through the status code. The server will then close the connection.
MAY have extras. MUST NOT have key. MUST NOT have value. 4 byte expiration time
Extra data for flush: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Expiration | +---------------+---------------+---------------+---------------+ Total 4 bytes
Flush the items in the cache now or some time in the future as specified by the expiration field. See the documentation of the textual protocol for the full description on how to specify the expiration time.
To flush the cache (delete all items) in two hours, the set the following values in the request
Flush request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x08 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x04 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x04 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x00 | 0x00 | 0x1c | 0x20 | +---------------+---------------+---------------+---------------+ Total 28 bytes (24 byte header, 4 byte body) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x08 Key length (2,3) : 0x0000 Extra length (4) : 0x04 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000004 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : Expiry (24-27): 0x00001c20 Key : None Value : None
The response-packet contains no extra data, and the result of the operation is signaled through the status code.
MUST NOT have extras. MUST NOT have key. MUST NOT have value. Used as a keep alive. Flushes outstanding getq/getkq's.
Noop request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x0a | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x0a Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value : None
The response-packet contains no extra data, and the result of the operation is signaled through the status code.
MUST NOT have extras. MUST NOT have key. MUST NOT have value. Request the server version. The server responds with a packet containing the version string in the body with the following format: "x.y.z"
Version request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x0b | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x0b Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None
Version response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x0b | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x31 ('1') | 0x2e ('.') | 0x33 ('3') | 0x2e ('.') | +---------------+---------------+---------------+---------------+ 28| 0x31 ('1') | +---------------+ Total 29 bytes (24 byte header, 5 byte body) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x0b Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000005 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value : Textual string "1.3.1"
MUST NOT have extras. MUST have key. MUST have value. These commands will either append or prepend the specified value to the requested key.
The following example appends '!' to the 'Hello' key.
Append request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x0e | 0x00 | 0x05 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x06 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x48 ('H') | 0x65 ('e') | 0x6c ('l') | 0x6c ('l') | +---------------+---------------+---------------+---------------+ 28| 0x6f ('o') | 0x21 ('!') | +---------------+---------------+ Total 30 bytes (24 byte header, 5 byte key, 1 byte value) Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x0e Key length (2,3) : 0x0005 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000006 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key (24-28): The textual string "Hello" Value (29) : "!"
The response-packet contains no extra data, and the result of the operation is signaled through the status code.
MUST NOT have extras. MAY have key. MUST NOT have value. Request server statistics. Without a key specified the server will respond with a "default" set of statistics information. Each piece of statistical information is returned in its own packet (key contains the name of the statistical item and the body contains the value in ASCII format). The sequence of return packets is terminated with a packet that contains no key and no value.
The following example requests all statistics from the server
Stat request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x80 | 0x10 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ Total 24 bytes Field (offset) (value) Magic (0) : 0x80 Opcode (1) : 0x10 Key length (2,3) : 0x0000 Extra length (4) : 0x00 Data type (5) : 0x00 Reserved (6,7) : 0x0000 Total body (8-11) : 0x00000000 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : None Value : None
The server will send each value in a separate packet with an "empty" packet (no key / no value) to terminate the sequence. Each of the response packets look like the following example:
Stat response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| 0x81 | 0x10 | 0x00 | 0x03 | +---------------+---------------+---------------+---------------+ 4| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 8| 0x00 | 0x00 | 0x00 | 0x07 | +---------------+---------------+---------------+---------------+ 12| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 16| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 20| 0x00 | 0x00 | 0x00 | 0x00 | +---------------+---------------+---------------+---------------+ 24| 0x70 ('p') | 0x69 ('i') | 0x64 ('d') | 0x33 ('3') | +---------------+---------------+---------------+---------------+ 28| 0x30 ('0') | 0x37 ('7') | 0x38 ('8') | +---------------+---------------+---------------+ Total 31 bytes (24 byte header, 3 byte key, 4 byte body) Field (offset) (value) Magic (0) : 0x81 Opcode (1) : 0x10 Key length (2,3) : 0x0003 Extra length (4) : 0x00 Data type (5) : 0x00 Status (6,7) : 0x0000 Total body (8-11) : 0x00000007 Opaque (12-15): 0x00000000 CAS (16-23): 0x0000000000000000 Extras : None Key : The textual string "pid" Value : The textual string "3078"
Memcache has few authentication and no security layers whatsoever. It is RECOMMENDED that memcache be deployed strictly on closed, protected, back-end networks within a single data center, within a single cluster of servers, or even on a single host, providing shared caching for multiple applications. Memcache MUST NOT be made available on a public network. SASL is supported as an authentication mechanism. See the wiki for more information.
LJ NEEDS MOAR SPEED Danga Interactive http://www.livejournal.com/
Thanks to Brad Fitzpatrick, Anatoly Vorobey, Steven Grimm, and Dustin Sallings, for their work on the memcached server. Thanks to Sean Chittenden, Jonathan Steinert, Brian Aker, Evan Martin, Nathan Neulinger, Eric Hodel, Michael Johnson, Paul Querna, Jamie McCarthy, Philip Neustrom, Andrew O'Brien, Josh Rotenberg, Robin H. Johnson, Tim Yardley, Paolo Borelli, Eli Bingham, Jean-Francois Bustarret, Paul G, Paul Lindner, Alan Kasindorf, Chris Goffinet, Tomash Brechko, and others for their work reporting bugs and maintaining memcached client libraries and bindings in many languages.
memcached-1.5.22/doc/protocol-binary-range.txt0000664000175000017500000002400213615454046016163 00000000000000 Network Working Group Aaron Stone, Ed. Internet-Draft Six Apart, Ltd. Intended status: Informational December 14, 2007 Expires: June 16, 2008 Memcache Binary Protocol: Extensions for UDP draft-stone-memcache-udp-01 Abstract This memo explains extensions to the memcache binary protocol for use in a UDP environment. Memcache is a high performance key-value cache. It is intentionally a dumb cache, optimized for speed only. Applications using memcache do not rely on it for data -- a persistent database with guaranteed reliability is strongly recommended -- but applications can run much faster when cached data is available in memcache. Status of This Memo This Internet-Draft is submitted in full conformance with the provisions of BCP 78 and BCP 79. Internet-Drafts are working documents of the Internet Engineering Task Force (IETF). Note that other groups may also distribute working documents as Internet-Drafts. The list of current Internet- Drafts is at http://datatracker.ietf.org/drafts/current/. Internet-Drafts are draft documents valid for a maximum of six months and may be updated, replaced, or obsoleted by other documents at any time. It is inappropriate to use Internet-Drafts as reference material or to cite them other than as "work in progress." This Internet-Draft will expire on June 16, 2008. Copyright Notice Copyright (c) 2007 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must Aaron Stone Expires June 16, 2008 [Page 1] Internet-Draft Memcache Over UDP December 2007 include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 1.1. Conventions Used In This Document . . . . . . . . . . . . 2 2. Defined Values . . . . . . . . . . . . . . . . . . . . . . . 3 2.1. Magic Byte . . . . . . . . . . . . . . . . . . . . . . . 3 2.2. Response Status . . . . . . . . . . . . . . . . . . . . . 3 2.3. Command Opcodes . . . . . . . . . . . . . . . . . . . . . 3 2.4. Data Types . . . . . . . . . . . . . . . . . . . . . . . 3 3. Commands . . . . . . . . . . . . . . . . . . . . . . . . . . 3 3.1. Get Response . . . . . . . . . . . . . . . . . . . . . . 3 3.2. Get Range Request . . . . . . . . . . . . . . . . . . . . 4 3.3. Get Range Response . . . . . . . . . . . . . . . . . . . 4 4. Security Considerations . . . . . . . . . . . . . . . . . . . 5 5. Normative References . . . . . . . . . . . . . . . . . . . . 5 Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 5 1. Introduction Memcache is a high performance key-value cache. It is intentionally a dumb cache, optimized for speed only. Applications using memcache do not rely on it for data -- a persistent database with guaranteed reliability is strongly recommended -- but applications can run much faster when cached data is available in memcache. Sites may find that, due to their network architecture or application usage patterns, the stateless [UDP] protocol better suits their needs. This document provides extensions and descriptions of use of the memcache protocol [MEMCACHE] in a UDP environment. It is a goal of this document to provide sufficient information in each UDP packet as to avoid any requirement for statefulness on the part of the server nor significant caching of outstanding packets on the part of the client. 1.1. Conventions Used In This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [KEYWORDS]. Aaron Stone Expires June 16, 2008 [Page 2] Internet-Draft Memcache Over UDP December 2007 2. Defined Values 2.1. Magic Byte The magic bytes remains the same as in [MEMCACHE]. 2.2. Response Status Additional status values: 0x0004 Value is larger than a single response packet 2.3. Command Opcodes Additional opcode values: 0x0C Get Range 0x0D Set Range 2.4. Data Types There are no new data types in this extension. 3. Commands 3.1. Get Response This section extends the behavior of the Get and GetQ commands as described in [MEMCACHE]. When a Get or GetQ request is made via UDP, and the value of the key for which the request was made is larger than can be placed into a single UDP packet (noting that the protocol header must also be counted), a Get Range response packet MUST be sent instead of the Get response packet. In this instance: 1. The Status field of the response header MUST be 0x0004. 2. The Offset field of the GetR response extras MUST be 0. 3. The Length field of the GetR response extras, and the data contained in the Value field of the packet, SHOULD be the maximum allowed length of a UDP packet, less the space required by the header and extras; however it MAY be any amount below this maximum. 4. The Total value length field of the response extras MUST be the actual length of the complete value. The client, upon receipt of a Get Range response bearing Status 0x004 and a Message ID corresponding to its Get request, shall then know Aaron Stone Expires June 16, 2008 [Page 3] Internet-Draft Memcache Over UDP December 2007 that it has received only the first portion of the value. The client MAY choose to request the remaining portion of the value by sending one or more Get Range requests. 3.2. Get Range Request The Get Range request is primarily intended for use over a UDP transport to request byte ranges of the value for a key. In the event that the Data version check fails to match that of the key, an error MUST be returned. Extra data for get range request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Flags | +---------------+---------------+---------------+---------------+ 4| Data version check | | | +---------------+---------------+---------------+---------------+ 12| Offset | +---------------+---------------+---------------+---------------+ 16| Length | +---------------+---------------+---------------+---------------+ Total 20 bytes 3.3. Get Range Response The Get Range request is primarily intended for use over a UDP transport to indicate the location of the bytes of the value for a key contained in a given packet. A client receives enough information in each Get Range extras to construct an appropriately sized buffer in its own memory and blindly insert the contents of the packet at the given byte offset. Aaron Stone Expires June 16, 2008 [Page 4] Internet-Draft Memcache Over UDP December 2007 Extra data for get range response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Flags | +---------------+---------------+---------------+---------------+ 4| Data version check | | | +---------------+---------------+---------------+---------------+ 12| Offset | +---------------+---------------+---------------+---------------+ 16| Length | +---------------+---------------+---------------+---------------+ 20| Total value length | +---------------+---------------+---------------+---------------+ Total 24 bytes 4. Security Considerations This document does not introduce any new security considerations beyond those discussed in [MEMCACHE]. 5. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [MEMCACHE] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [UDP] Postel, J., "User Datagram Protocol", STD 6, RFC 768, August 1980. Author's Address Aaron Stone (editor) Six Apart, Ltd. 548 4th Street San Francisco, CA 94107 USA Email: aaron@serendipity.palo-alto.ca.us Aaron Stone Expires June 16, 2008 [Page 5] memcached-1.5.22/doc/Doxyfile0000664000175000017500000014452113461170555012723 00000000000000# Doxyfile 1.5.2 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project # # All text after a hash (#) is considered a comment and will be ignored # The format is: # TAG = value [value, ...] # For lists items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (" ") #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the config file that # follow. The default is UTF-8 which is also the encoding used for all text before # the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into # libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of # possible encodings. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded # by quotes) that should identify the project. PROJECT_NAME = memcached # The PROJECT_NUMBER tag can be used to enter a project or revision number. # This could be handy for archiving the generated documentation or # if some version control system is used. PROJECT_NUMBER = 0.8 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) # base path where the generated documentation will be put. # If a relative path is entered, it will be relative to the location # where doxygen was started. If left blank the current directory will be used. OUTPUT_DIRECTORY = doxygen # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create # 4096 sub-directories (in 2 levels) under the output directory of each output # format and will distribute the generated files over these directories. # Enabling this option can be useful when feeding doxygen a huge amount of # source files, where putting all generated files in the same directory would # otherwise cause performance problems for the file system. CREATE_SUBDIRS = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # The default language is English, other supported languages are: # Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, # Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, # Italian, Japanese, Japanese-en (Japanese with English messages), Korean, # Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, # Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. OUTPUT_LANGUAGE = English # If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will # include brief member descriptions after the members that are listed in # the file and class documentation (similar to JavaDoc). # Set to NO to disable this. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend # the brief description of a member or function before the detailed description. # Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. REPEAT_BRIEF = NO # This tag implements a quasi-intelligent brief description abbreviator # that is used to form the text in various listings. Each string # in this list, if found as the leading text of the brief description, will be # stripped from the text and the result after processing the whole list, is # used as the annotated text. Otherwise, the brief description is used as-is. # If left blank, the following values are used ("$name" is automatically # replaced with the name of the entity): "The $name class" "The $name widget" # "The $name file" "is" "provides" "specifies" "contains" # "represents" "a" "an" "the" ABBREVIATE_BRIEF = # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # Doxygen will generate a detailed section even if there is only a brief # description. ALWAYS_DETAILED_SEC = YES # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full # path before files name in the file list and in the header files. If set # to NO the shortest path that makes the file name unique will be used. FULL_PATH_NAMES = NO # If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag # can be used to strip a user-defined part of the path. Stripping is # only done if one of the specified strings matches the left-hand part of # the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the # path to strip. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of # the path mentioned in the documentation of a class, which tells # the reader which header file to include in order to use a class. # If left blank only the name of the header file containing the class # definition is used. Otherwise one should specify the include paths that # are normally passed to the compiler using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter # (but less readable) file names. This can be useful is your file systems # doesn't support long names like on DOS, Mac, or CD-ROM. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen # will interpret the first line (until the first dot) of a JavaDoc-style # comment as the brief description. If set to NO, the JavaDoc # comments will behave just like the Qt-style comments (thus requiring an # explicit @brief command for a brief description. JAVADOC_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen # treat a multi-line C++ special comment block (i.e. a block of //! or /// # comments) as a brief description. This used to be the default behaviour. # The new default is to treat a multi-line C++ comment block as a detailed # description. Set this tag to YES if you prefer the old behaviour instead. MULTILINE_CPP_IS_BRIEF = NO # If the DETAILS_AT_TOP tag is set to YES then Doxygen # will output the detailed description near the top, like JavaDoc. # If set to NO, the detailed description appears after the member # documentation. DETAILS_AT_TOP = NO # If the INHERIT_DOCS tag is set to YES (the default) then an undocumented # member inherits the documentation from any documented member that it # re-implements. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce # a new page for each member. If set to NO, the documentation of a member will # be part of the file/class/namespace that contains it. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. # Doxygen uses this value to replace tabs by spaces in code fragments. TAB_SIZE = 8 # This tag can be used to specify a number of aliases that acts # as commands in the documentation. An alias has the form "name=value". # For example adding "sideeffect=\par Side Effects:\n" will allow you to # put the command \sideeffect (or @sideeffect) in the documentation, which # will result in a user-defined paragraph with heading "Side Effects:". # You can put \n's in the value part of an alias to insert newlines. ALIASES = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C # sources only. Doxygen will then generate output that is more tailored for C. # For instance, some of the names that are used will be different. The list # of all members will be omitted, etc. OPTIMIZE_OUTPUT_FOR_C = NO # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java # sources only. Doxygen will then generate output that is more tailored for Java. # For instance, namespaces will be presented as packages, qualified scopes # will look different, etc. OPTIMIZE_OUTPUT_JAVA = NO # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to # include (a tag file for) the STL sources as input, then you should # set this tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); v.s. # func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. CPP_CLI_SUPPORT = NO # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES, then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. DISTRIBUTE_GROUP_DOC = NO # Set the SUBGROUPING tag to YES (the default) to allow class member groups of # the same type (for instance a group of public functions) to be put as a # subgroup of that type (e.g. under the Public Functions section). Set it to # NO to prevent subgrouping. Alternatively, this can be done per class using # the \nosubgrouping command. SUBGROUPING = YES #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in # documentation are documented, even if no documentation was available. # Private class members and static file members will be hidden unless # the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES EXTRACT_ALL = YES # If the EXTRACT_PRIVATE tag is set to YES all private members of a class # will be included in the documentation. EXTRACT_PRIVATE = NO # If the EXTRACT_STATIC tag is set to YES all static members of a file # will be included in the documentation. EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) # defined locally in source files will be included in the documentation. # If set to NO only classes defined in header files are included. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. When set to YES local # methods, which are defined in the implementation section but not in # the interface are included in the documentation. # If set to NO (the default) only methods in the interface are included. EXTRACT_LOCAL_METHODS = NO # If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all # undocumented members of documented classes, files or namespaces. # If set to NO (the default) these members will be included in the # various overviews, but no documentation section is generated. # This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. # If set to NO (the default) these classes will be included in the various # overviews. This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all # friend (class|struct|union) declarations. # If set to NO (the default) these declarations will be included in the # documentation. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any # documentation blocks found inside the body of a function. # If set to NO (the default) these blocks will be appended to the # function's detailed documentation block. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation # that is typed after a \internal command is included. If the tag is set # to NO (the default) then the documentation will be excluded. # Set it to YES to include the internal documentation. INTERNAL_DOCS = YES # If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate # file names in lower-case letters. If set to YES upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. CASE_SENSE_NAMES = YES # If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen # will show members with their full class and namespace scopes in the # documentation. If set to YES the scope will be hidden. HIDE_SCOPE_NAMES = NO # If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen # will put a list of the files that are included by a file in the documentation # of that file. SHOW_INCLUDE_FILES = YES # If the INLINE_INFO tag is set to YES (the default) then a tag [inline] # is inserted in the documentation for inline members. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen # will sort the (detailed) documentation of file and class members # alphabetically by member name. If set to NO the members will appear in # declaration order. SORT_MEMBER_DOCS = NO # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the # brief documentation of file, namespace and class members alphabetically # by member name. If set to NO (the default) the members will appear in # declaration order. SORT_BRIEF_DOCS = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be # sorted by fully-qualified names, including namespaces. If set to # NO (the default), the class list will be sorted only by class name, # not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the # alphabetical list. SORT_BY_SCOPE_NAME = NO # The GENERATE_TODOLIST tag can be used to enable (YES) or # disable (NO) the todo list. This list is created by putting \todo # commands in the documentation. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable (YES) or # disable (NO) the test list. This list is created by putting \test # commands in the documentation. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable (YES) or # disable (NO) the bug list. This list is created by putting \bug # commands in the documentation. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or # disable (NO) the deprecated list. This list is created by putting # \deprecated commands in the documentation. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional # documentation sections, marked by \if sectionname ... \endif. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines # the initial value of a variable or define consists of for it to appear in # the documentation. If the initializer consists of more lines than specified # here it will be hidden. Use a value of 0 to hide initializers completely. # The appearance of the initializer of individual variables and defines in the # documentation can be controlled using \showinitializer or \hideinitializer # command in the documentation regardless of this setting. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated # at the bottom of the documentation of classes and structs. If set to YES the # list will mention the files that were used to generate the documentation. SHOW_USED_FILES = YES # If the sources in your project are distributed over multiple directories # then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy # in the documentation. The default is NO. SHOW_DIRECTORIES = NO # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from the # version control system). Doxygen will invoke the program by executing (via # popen()) the command , where is the value of # the FILE_VERSION_FILTER tag, and is the name of an input file # provided by doxygen. Whatever the program writes to standard output # is used as the file version. See the manual for examples. FILE_VERSION_FILTER = #--------------------------------------------------------------------------- # configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated # by doxygen. Possible values are YES and NO. If left blank NO is used. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated by doxygen. Possible values are YES and NO. If left blank # NO is used. WARNINGS = YES # If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings # for undocumented members. If EXTRACT_ALL is set to YES then this flag will # automatically be disabled. WARN_IF_UNDOCUMENTED = YES # If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some # parameters in a documented function, or documenting parameters that # don't exist or using markup commands wrongly. WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be enabled to get warnings for # functions that are documented, but have no documentation for their parameters # or return value. If set to NO (the default) doxygen will only warn about # wrong or incomplete parameter documentation, but not about the absence of # documentation. WARN_NO_PARAMDOC = NO # The WARN_FORMAT tag determines the format of the warning messages that # doxygen can produce. The string should contain the $file, $line, and $text # tags, which will be replaced by the file and line number from which the # warning originated and the warning text. Optionally the format may contain # $version, which will be replaced by the version of the file (if it could # be obtained via FILE_VERSION_FILTER) WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning # and error messages should be written. If left blank the output is written # to stderr. WARN_LOGFILE = #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag can be used to specify the files and/or directories that contain # documented source files. You may enter file names like "myfile.cpp" or # directories like "/usr/src/myproject". Separate the files or directories # with spaces. INPUT = .. # This tag can be used to specify the character encoding of the source files that # doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default # input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. # See http://www.gnu.org/software/libiconv for the list of possible encodings. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank the following patterns are tested: # *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py FILE_PATTERNS = *.h \ *.c # The RECURSIVE tag can be used to turn specify whether or not subdirectories # should be searched for input files as well. Possible values are YES and NO. # If left blank NO is used. RECURSIVE = NO # The EXCLUDE tag can be used to specify files and/or directories that should # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. EXCLUDE = config.h # The EXCLUDE_SYMLINKS tag can be used select whether or not files or # directories that are symbolic links (a Unix filesystem feature) are excluded # from the input. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. Note that the wildcards are matched # against the file with absolute path, so to exclude all test directories # for example use the pattern */test/* EXCLUDE_PATTERNS = # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the output. # The symbol name can be a fully qualified name, a word, or if the wildcard * is used, # a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or # directories that contain example code fragments that are included (see # the \include command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank all files are included. EXAMPLE_PATTERNS = # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude # commands irrespective of the value of the RECURSIVE tag. # Possible values are YES and NO. If left blank NO is used. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or # directories that contain image that are included in the documentation (see # the \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command , where # is the value of the INPUT_FILTER tag, and is the name of an # input file. Doxygen will then use the output that the filter program writes # to standard output. If FILTER_PATTERNS is specified, this tag will be # ignored. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: # pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further # info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER # is applied to all files. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER) will be used to filter the input files when producing source # files to browse (i.e. when SOURCE_BROWSER is set to YES). FILTER_SOURCE_FILES = NO #--------------------------------------------------------------------------- # configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will # be generated. Documented entities will be cross-referenced with these sources. # Note: To get rid of all source code in the generated output, make sure also # VERBATIM_HEADERS is set to NO. SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body # of functions and classes directly in the documentation. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct # doxygen to hide any special comment blocks from generated source code # fragments. Normal C and C++ comments will always remain visible. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES (the default) # then for each documented function all documented # functions referencing it will be listed. REFERENCED_BY_RELATION = YES # If the REFERENCES_RELATION tag is set to YES (the default) # then for each documented function all documented entities # called/used by that function will be listed. REFERENCES_RELATION = YES # If the REFERENCES_LINK_SOURCE tag is set to YES (the default) # and SOURCE_BROWSER tag is set to YES, then the hyperlinks from # functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will # link to the source code. Otherwise they will link to the documentation. REFERENCES_LINK_SOURCE = YES # If the USE_HTAGS tag is set to YES then the references to source code # will point to the HTML generated by the htags(1) tool instead of doxygen # built-in source browser. The htags tool is part of GNU's global source # tagging system (see http://www.gnu.org/software/global/global.html). You # will need version 4.8.6 or higher. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen # will generate a verbatim copy of the header file for each class for # which an include is specified. Set to NO to disable this. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index # of all compounds will be generated. Enable this if the project # contains a lot of classes, structs, unions or interfaces. ALPHABETICAL_INDEX = NO # If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then # the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns # in which this list will be split (can be a number in the range [1..20]) COLS_IN_ALPHA_INDEX = 5 # In case all classes in a project start with a common prefix, all # classes will be put under the same header in the alphabetical index. # The IGNORE_PREFIX tag can be used to specify one or more prefixes that # should be ignored while generating the index headers. IGNORE_PREFIX = #--------------------------------------------------------------------------- # configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES (the default) Doxygen will # generate HTML output. GENERATE_HTML = YES # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `html' will be used as the default path. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for # each generated HTML page (for example: .htm,.php,.asp). If it is left blank # doxygen will generate files with .html extension. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a personal HTML header for # each generated HTML page. If it is left blank doxygen will generate a # standard header. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a personal HTML footer for # each generated HTML page. If it is left blank doxygen will generate a # standard footer. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading # style sheet that is used by each HTML page. It can be used to # fine-tune the look of the HTML output. If the tag is left blank doxygen # will generate a default style sheet. Note that doxygen will try to copy # the style sheet file to the HTML output directory, so don't put your own # stylesheet in the HTML output directory as well, or it will be erased! HTML_STYLESHEET = # If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, # files or namespaces will be aligned in HTML using tables. If set to # NO a bullet list will be used. HTML_ALIGN_MEMBERS = YES # If the GENERATE_HTMLHELP tag is set to YES, additional index files # will be generated that can be used as input for tools like the # Microsoft HTML help workshop to generate a compressed HTML help file (.chm) # of the generated HTML documentation. GENERATE_HTMLHELP = NO # If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can # be used to specify the file name of the resulting .chm file. You # can add a path in front of the file if the result should not be # written to the html output directory. CHM_FILE = # If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can # be used to specify the location (absolute path including file name) of # the HTML help compiler (hhc.exe). If non-empty doxygen will try to run # the HTML help compiler on the generated index.hhp. HHC_LOCATION = # If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag # controls if a separate .chi index file is generated (YES) or that # it should be included in the master .chm file (NO). GENERATE_CHI = NO # If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag # controls whether a binary table of contents is generated (YES) or a # normal table of contents (NO) in the .chm file. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members # to the contents of the HTML help documentation and to the tree view. TOC_EXPAND = NO # The DISABLE_INDEX tag can be used to turn on/off the condensed index at # top of each HTML page. The value NO (the default) enables the index and # the value YES disables it. DISABLE_INDEX = NO # This tag can be used to set the number of enum values (range [1..20]) # that doxygen will group on one line in the generated HTML documentation. ENUM_VALUES_PER_LINE = 4 # If the GENERATE_TREEVIEW tag is set to YES, a side panel will be # generated containing a tree-like index structure (just like the one that # is generated for HTML Help). For this to work a browser that supports # JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, # Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are # probably better off using the HTML help feature. GENERATE_TREEVIEW = NO # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be # used to set the initial width (in pixels) of the frame in which the tree # is shown. TREEVIEW_WIDTH = 250 #--------------------------------------------------------------------------- # configuration options related to the LaTeX output #--------------------------------------------------------------------------- # If the GENERATE_LATEX tag is set to YES (the default) Doxygen will # generate Latex output. GENERATE_LATEX = NO # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `latex' will be used as the default path. LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. If left blank `latex' will be used as the default command name. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to # generate index for LaTeX. If left blank `makeindex' will be used as the # default command name. MAKEINDEX_CMD_NAME = makeindex # If the COMPACT_LATEX tag is set to YES Doxygen generates more compact # LaTeX documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_LATEX = NO # The PAPER_TYPE tag can be used to set the paper type that is used # by the printer. Possible values are: a4, a4wide, letter, legal and # executive. If left blank a4wide will be used. PAPER_TYPE = a4wide # The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX # packages that should be included in the LaTeX output. EXTRA_PACKAGES = # The LATEX_HEADER tag can be used to specify a personal LaTeX header for # the generated latex document. The header should contain everything until # the first chapter. If it is left blank doxygen will generate a # standard header. Notice: only use this tag if you know what you are doing! LATEX_HEADER = # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated # is prepared for conversion to pdf (using ps2pdf). The pdf file will # contain links (just like the HTML output) instead of page references # This makes the output suitable for online browsing using a pdf viewer. PDF_HYPERLINKS = NO # If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of # plain latex in the generated Makefile. Set this option to YES to get a # higher quality PDF documentation. USE_PDFLATEX = NO # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. # command to the generated LaTeX files. This will instruct LaTeX to keep # running if errors occur, instead of asking the user for help. # This option is also used when generating formulas in HTML. LATEX_BATCHMODE = NO # If LATEX_HIDE_INDICES is set to YES then doxygen will not # include the index chapters (such as File Index, Compound Index, etc.) # in the output. LATEX_HIDE_INDICES = NO #--------------------------------------------------------------------------- # configuration options related to the RTF output #--------------------------------------------------------------------------- # If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output # The RTF output is optimized for Word 97 and may not look very pretty with # other RTF readers or editors. GENERATE_RTF = NO # The RTF_OUTPUT tag is used to specify where the RTF docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `rtf' will be used as the default path. RTF_OUTPUT = rtf # If the COMPACT_RTF tag is set to YES Doxygen generates more compact # RTF documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_RTF = NO # If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated # will contain hyperlink fields. The RTF file will # contain links (just like the HTML output) instead of page references. # This makes the output suitable for online browsing using WORD or other # programs which support those fields. # Note: wordpad (write) and others do not support links. RTF_HYPERLINKS = NO # Load stylesheet definitions from file. Syntax is similar to doxygen's # config file, i.e. a series of assignments. You only have to provide # replacements, missing definitions are set to their default value. RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an rtf document. # Syntax is similar to doxygen's config file. RTF_EXTENSIONS_FILE = #--------------------------------------------------------------------------- # configuration options related to the man page output #--------------------------------------------------------------------------- # If the GENERATE_MAN tag is set to YES (the default) Doxygen will # generate man pages GENERATE_MAN = YES # The MAN_OUTPUT tag is used to specify where the man pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `man' will be used as the default path. MAN_OUTPUT = man # The MAN_EXTENSION tag determines the extension that is added to # the generated man pages (default is the subroutine's section .3) MAN_EXTENSION = .3 # If the MAN_LINKS tag is set to YES and Doxygen generates man output, # then it will generate one additional man file for each entity # documented in the real man page(s). These additional files # only source the real man page, but without them the man command # would be unable to find the correct page. The default is NO. MAN_LINKS = NO #--------------------------------------------------------------------------- # configuration options related to the XML output #--------------------------------------------------------------------------- # If the GENERATE_XML tag is set to YES Doxygen will # generate an XML file that captures the structure of # the code including all documentation. GENERATE_XML = NO # The XML_OUTPUT tag is used to specify where the XML pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `xml' will be used as the default path. XML_OUTPUT = xml # The XML_SCHEMA tag can be used to specify an XML schema, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_SCHEMA = # The XML_DTD tag can be used to specify an XML DTD, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_DTD = # If the XML_PROGRAMLISTING tag is set to YES Doxygen will # dump the program listings (including syntax highlighting # and cross-referencing information) to the XML output. Note that # enabling this will significantly increase the size of the XML output. XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- # configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will # generate an AutoGen Definitions (see autogen.sf.net) file # that captures the structure of the code including all # documentation. Note that this feature is still experimental # and incomplete at the moment. GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # configuration options related to the Perl module output #--------------------------------------------------------------------------- # If the GENERATE_PERLMOD tag is set to YES Doxygen will # generate a Perl module file that captures the structure of # the code including all documentation. Note that this # feature is still experimental and incomplete at the # moment. GENERATE_PERLMOD = NO # If the PERLMOD_LATEX tag is set to YES Doxygen will generate # the necessary Makefile rules, Perl scripts and LaTeX code to be able # to generate PDF and DVI output from the Perl module output. PERLMOD_LATEX = NO # If the PERLMOD_PRETTY tag is set to YES the Perl module output will be # nicely formatted so it can be parsed by a human reader. This is useful # if you want to understand what is going on. On the other hand, if this # tag is set to NO the size of the Perl module output will be much smaller # and Perl will parse it just the same. PERLMOD_PRETTY = YES # The names of the make variables in the generated doxyrules.make file # are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. # This is useful so different doxyrules.make files included by the same # Makefile don't overwrite each other's variables. PERLMOD_MAKEVAR_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- # If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will # evaluate all C-preprocessor directives found in the sources and include # files. ENABLE_PREPROCESSING = YES # If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro # names in the source code. If set to NO (the default) only conditional # compilation will be performed. Macro expansion can be done in a controlled # way by setting EXPAND_ONLY_PREDEF to YES. MACRO_EXPANSION = NO # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES # then the macro expansion is limited to the macros specified with the # PREDEFINED and EXPAND_AS_DEFINED tags. EXPAND_ONLY_PREDEF = NO # If the SEARCH_INCLUDES tag is set to YES (the default) the includes files # in the INCLUDE_PATH (see below) will be search if a #include is found. SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by # the preprocessor. INCLUDE_PATH = # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard # patterns (like *.h and *.hpp) to filter out the header-files in the # directories. If left blank, the patterns specified with FILE_PATTERNS will # be used. INCLUDE_FILE_PATTERNS = # The PREDEFINED tag can be used to specify one or more macro names that # are defined before the preprocessor is started (similar to the -D option of # gcc). The argument of the tag is a list of macros of the form: name # or name=definition (no spaces). If the definition and the = are # omitted =1 is assumed. To prevent a macro definition from being # undefined via #undef or recursively expanded use the := operator # instead of the = operator. PREDEFINED = # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then # this tag can be used to specify a list of macro names that should be expanded. # The macro definition that is found in the sources will be used. # Use the PREDEFINED tag if you want to use a different macro definition. EXPAND_AS_DEFINED = # If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then # doxygen's preprocessor will remove all function-like macros that are alone # on a line, have an all uppercase name, and do not end with a semicolon. Such # function macros are typically used for boiler-plate code, and will confuse # the parser if not removed. SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration::additions related to external references #--------------------------------------------------------------------------- # The TAGFILES option can be used to specify one or more tagfiles. # Optionally an initial location of the external documentation # can be added for each tagfile. The format of a tag file without # this location is as follows: # TAGFILES = file1 file2 ... # Adding location for the tag files is done as follows: # TAGFILES = file1=loc1 "file2 = loc2" ... # where "loc1" and "loc2" can be relative or absolute paths or # URLs. If a location is present for each tag, the installdox tool # does not have to be run to correct the links. # Note that each tag file must have a unique name # (where the name does NOT include the path) # If a tag file is not located in the directory in which doxygen # is run, you must also specify the path to the tagfile here. TAGFILES = # When a file name is specified after GENERATE_TAGFILE, doxygen will create # a tag file that is based on the input files it reads. GENERATE_TAGFILE = # If the ALLEXTERNALS tag is set to YES all external classes will be listed # in the class index. If set to NO only the inherited external classes # will be listed. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed # in the modules index. If set to NO, only the current project's groups will # be listed. EXTERNAL_GROUPS = YES # The PERL_PATH should be the absolute path and name of the perl script # interpreter (i.e. the result of `which perl'). PERL_PATH = /usr/bin/perl #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- # If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will # generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base # or super classes. Setting the tag to NO turns the diagrams off. Note that # this option is superseded by the HAVE_DOT option below. This is only a # fallback. It is recommended to install and use dot, since it yields more # powerful graphs. CLASS_DIAGRAMS = YES # You can define message sequence charts within doxygen comments using the \msc # command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to # produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to # specify the directory where the mscgen tool resides. If left empty the tool is assumed to # be found in the default search path. MSCGEN_PATH = # If set to YES, the inheritance and collaboration graphs will hide # inheritance and usage relations if the target is undocumented # or is not a class. HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz, a graph visualization # toolkit from AT&T and Lucent Bell Labs. The other options in this section # have no effect if this option is set to NO (the default) HAVE_DOT = NO # If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect inheritance relations. Setting this tag to YES will force the # the CLASS_DIAGRAMS tag to NO. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect implementation dependencies (inheritance, containment, and # class references variables) of the class with other documented classes. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen # will generate a graph for groups, showing the direct groups dependencies GROUP_GRAPHS = YES # If the UML_LOOK tag is set to YES doxygen will generate inheritance and # collaboration diagrams in a style similar to the OMG's Unified Modeling # Language. UML_LOOK = NO # If set to YES, the inheritance and collaboration graphs will show the # relations between templates and their instances. TEMPLATE_RELATIONS = NO # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT # tags are set to YES then doxygen will generate a graph for each documented # file showing the direct and indirect include dependencies of the file with # other documented files. INCLUDE_GRAPH = YES # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and # HAVE_DOT tags are set to YES then doxygen will generate a graph for each # documented header file showing the documented files that directly or # indirectly include this file. INCLUDED_BY_GRAPH = YES # If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will # generate a call dependency graph for every global function or class method. # Note that enabling this option will significantly increase the time of a run. # So in most cases it will be better to enable call graphs for selected # functions only using the \callgraph command. CALL_GRAPH = NO # If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will # generate a caller dependency graph for every global function or class method. # Note that enabling this option will significantly increase the time of a run. # So in most cases it will be better to enable caller graphs for selected # functions only using the \callergraph command. CALLER_GRAPH = NO # If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen # will graphical hierarchy of all classes instead of a textual one. GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES # then doxygen will show the dependencies a directory has on other directories # in a graphical way. The dependency relations are determined by the #include # relations between the files in the directories. DIRECTORY_GRAPH = YES # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. Possible values are png, jpg, or gif # If left blank png will be used. DOT_IMAGE_FORMAT = png # The tag DOT_PATH can be used to specify the path where the dot tool can be # found. If left blank, it is assumed the dot tool can be found in the path. DOT_PATH = # The DOTFILE_DIRS tag can be used to specify one or more directories that # contain dot files that are included in the documentation (see the # \dotfile command). DOTFILE_DIRS = # The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of # nodes that will be shown in the graph. If the number of nodes in a graph # becomes larger than this value, doxygen will truncate the graph, which is # visualized by representing a node as a red box. Note that doxygen will always # show the root nodes and its direct children regardless of this setting. DOT_GRAPH_MAX_NODES = 50 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent # background. This is disabled by default, which results in a white background. # Warning: Depending on the platform used, enabling this option may lead to # badly anti-aliased labels on the edges of a graph (i.e. they become hard to # read). DOT_TRANSPARENT = NO # Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) # support this, this feature is disabled by default. DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will # generate a legend page explaining the meaning of the various boxes and # arrows in the dot generated graphs. GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES (the default) Doxygen will # remove the intermediate dot files that are used to generate # the various graphs. DOT_CLEANUP = YES #--------------------------------------------------------------------------- # Configuration::additions related to the search engine #--------------------------------------------------------------------------- # The SEARCHENGINE tag specifies whether or not a search engine should be # used. If set to NO the values of all tags below this one will be ignored. SEARCHENGINE = NO memcached-1.5.22/doc/storage.txt0000664000175000017500000001402113461170555013411 00000000000000Storage system notes -------------------- extstore.h defines the API. extstore_write() is a synchronous call which memcpy's the input buffer into a write buffer for an active page. A failure is not usually a hard failure, but indicates caller can try again another time. IE: it might be busy freeing pages or assigning new ones. As of this writing the write() implementation doesn't have an internal loop, so it can give spurious failures (good for testing integration) extstore_read() is an asynchronous call which takes a stack of IO objects and adds it to the end of a queue. It then signals the IO thread to run. Once an IO stack is submitted the caller must not touch the submitted objects anymore (they are relinked internally). extstore_delete() is a synchronous call which informs the storage engine an item has been removed from that page. It's important to call this as items are actively deleted or passively reaped due to TTL expiration. This allows the engine to intelligently reclaim pages. The IO threads execute each object in turn (or in bulk of running in the future libaio mode). Callbacks are issued from the IO threads. It's thus important to keep processing to a minimum. Callbacks may be issued out of order, and it is the caller's responsibility to know when its stack has been fully processed so it may reclaim the memory. With DIRECT_IO support, buffers submitted for read/write will need to be aligned with posix_memalign() or similar. Buckets ------- During extstore_init(), a number of active buckets is specified. Pages are handled overall as a global pool, but writes can be redirected to specific active pages. This allows a lot of flexibility, ie: 1) an idea of "high TTL" and "low TTL" being two buckets. TTL < 86400 goes into bucket 0, rest into bucket 1. Co-locating low TTL items means those pages can reach zero objects and free up more easily. 2) Extended: "low TTL" is one bucket, and then one bucket per slab class. If TTL's are low, mixed sized objects can go together as they are likely to expire before cycling out of flash (depending on workload, of course). For higher TTL items, pages are stored on chunk barriers. This means less space is wasted as items should fit nearly exactly into write buffers and pages. It also means you can blindly read items back if the system wants to free a page and we can indicate to the caller somehow which pages are up for probation. ie; issue a read against page 3 version 1 for byte range 0->1MB, then chunk and look up objects. Then read next 1MB chunk/etc. If there's anything we want to keep, pull it back into RAM before pages is freed. Pages are assigned into buckets on demand, so if you make 30 but use 1 there will only be a single active page with write buffers. Memcached integration --------------------- With the POC: items.c's lru_maintainer_thread calls writes to storage if all memory has been allocated out to slab classes, and there is less than an amount of memory free. Original objects are swapped with items marked with ITEM_HDR flag. an ITEM_HDR contains copies of the original key and most of the header data. The ITEM_data() section of an ITEM_HDR object contains (item_hdr *), which describes enough information to retrieve the original object from storage. To get best performance is important that reads can be deeply pipelined. As much processing as possible is done ahead of time, IO's are submitted, and once IO's are done processing a minimal amount of code is executed before transmit() is possible. This should amortize the amount of latency incurred by hopping threads and waiting on IO. Recaching --------- If a header is hit twice overall, and the second time within ~60s of the first time, it will have a chance of getting recached. "recache_rate" is a simple "counter % rate == 0" check. Setting to 1000 means one out of every 1000 instances of an item being hit twice within ~60s it will be recached into memory. Very hot items will get pulled out of storage relatively quickly. Compaction ---------- A target fragmentation limit is set: "0.9", meaning "run compactions if pages exist which have less than 90% of their bytes used". This value is slewed based on the number of free pages in the system, and activates when half of the pages used. The percentage of free pages is multiplied against the target fragmentation limit, ie: limit of 0.9: 50% of pages free -> 0.9 * 0.5 -> 0.45%. If a page is 64 megabytes, pages with less than 28.8 megabytes used would be targeted for compaction. If 0 pges are free, anything less than 90% used is targeted, which means it has to rewrite 10 pages to free one page. In memcached's integration, a second bucket is used for objects rewritten via the compactor. Potentially objects around long enough to get compacted might continue to stick around, so co-locating them could reduce fragmentation work. If an exclusive lock is made on a valid object header, the flash locations are rewritten directly in the object. As of this writing, if an object header is busy for some reason, the write is dropped (COW needs to be implemented). This is an unlikely scenario however. Objects are read back along the boundaries of a write buffer. If an 8 meg write buffer is used, 8 megs are read back at once and iterated for objects. This needs a fair amount of tuning, possibly more throttling. It will still evict pages if the compactor gets behind. TODO ---- Sharing my broad TODO items into here. While doing the work they get split up more into local notes. Adding this so others can follow along: (a bunch of the TODO has been completed and removed) - O_DIRECT support - libaio support (requires O_DIRECT) - JBOD support (not in first pass) - 1-2 active pages per device. potentially dedicated IO threads per device. with a RAID setup you risk any individual disk doing a GC pause stalling all writes. could also simply rotate devices on a per-bucket basis. on memcached end: - O_DIRECT support; mostly memalign pages, but also making chunks grow aligned to sector sizes once they are >= a single sector. - specify storage size by MB/G/T/etc instead of page count memcached-1.5.22/doc/protocol-binary-range.xml0000664000175000017500000002211113461170555016142 00000000000000 Memcache Binary Protocol: Extensions for UDP Six Apart, Ltd.
548 4th Street San Francisco CA 94107 USA aaron@serendipity.palo-alto.ca.us
Applications memcache memcached cache udp This memo explains extensions to the memcache binary protocol for use in a UDP environment. Memcache is a high performance key-value cache. It is intentionally a dumb cache, optimized for speed only. Applications using memcache do not rely on it for data -- a persistent database with guaranteed reliability is strongly recommended -- but applications can run much faster when cached data is available in memcache.
Memcache is a high performance key-value cache. It is intentionally a dumb cache, optimized for speed only. Applications using memcache do not rely on it for data -- a persistent database with guaranteed reliability is strongly recommended -- but applications can run much faster when cached data is available in memcache. Sites may find that, due to their network architecture or application usage patterns, the stateless protocol better suits their needs. This document provides extensions and descriptions of use of the memcache protocol in a UDP environment. It is a goal of this document to provide sufficient information in each UDP packet as to avoid any requirement for statefulness on the part of the server nor significant caching of outstanding packets on the part of the client.
The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in .
The magic bytes remains the same as in .
Additional status values: Value is larger than a single response packet
Additional opcode values: Get Range Set Range
There are no new data types in this extension.
This section extends the behavior of the Get and GetQ commands as described in . When a Get or GetQ request is made via UDP, and the value of the key for which the request was made is larger than can be placed into a single UDP packet (noting that the protocol header must also be counted), a Get Range response packet MUST be sent instead of the Get response packet. In this instance: The Status field of the response header MUST be 0x0004. The Offset field of the GetR response extras MUST be 0. The Length field of the GetR response extras, and the data contained in the Value field of the packet, SHOULD be the maximum allowed length of a UDP packet, less the space required by the header and extras; however it MAY be any amount below this maximum. The Total value length field of the response extras MUST be the actual length of the complete value. The client, upon receipt of a Get Range response bearing Status 0x004 and a Message ID corresponding to its Get request, shall then know that it has received only the first portion of the value. The client MAY choose to request the remaining portion of the value by sending one or more Get Range requests.
The Get Range request is primarily intended for use over a UDP transport to request byte ranges of the value for a key. In the event that the Data version check fails to match that of the key, an error MUST be returned.
Extra data for get range request: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Flags | +---------------+---------------+---------------+---------------+ 4| Data version check | | | +---------------+---------------+---------------+---------------+ 12| Offset | +---------------+---------------+---------------+---------------+ 16| Length | +---------------+---------------+---------------+---------------+ Total 20 bytes
The Get Range request is primarily intended for use over a UDP transport to indicate the location of the bytes of the value for a key contained in a given packet. A client receives enough information in each Get Range extras to construct an appropriately sized buffer in its own memory and blindly insert the contents of the packet at the given byte offset.
Extra data for get range response: Byte/ 0 | 1 | 2 | 3 | / | | | | |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +---------------+---------------+---------------+---------------+ 0| Flags | +---------------+---------------+---------------+---------------+ 4| Data version check | | | +---------------+---------------+---------------+---------------+ 12| Offset | +---------------+---------------+---------------+---------------+ 16| Length | +---------------+---------------+---------------+---------------+ 20| Total value length | +---------------+---------------+---------------+---------------+ Total 24 bytes
This document does not introduce any new security considerations beyond those discussed in .
memcached-1.5.22/doc/new_lru.txt0000664000175000017500000000464513461170555013433 00000000000000In versions new enough to have the `-o lru_maintainer` option, a new LRU mechanic is available. Previously: Each slab class has an independent doubly-linked list comprising its LRU. Items are pulled from the bottom and either reclaimed or evicted as needed. Now, enabling `-o lru_maintainer` changes all of the behavior below: * LRU's are now split between HOT, WARM, and COLD LRU's. New items enter the HOT LRU. * Items hit at least twice are considered active. * LRU updates only happen as items reach the bottom of an LRU. If active in HOT, move to WARM, if active in WARM, stay in WARM. If active in COLD, move to WARM. The exception is that items active in COLD are immediately moved to WARM. * HOT/WARM each capped at N% of memory available for that slab class. COLD is uncapped (by default, as of this writing). * HOT/WARM are also age capped by ratios of COLD's age. IE: HOT is capped at N% of memory, or 10% of the age of COLD, whichever comes first. * Items flow from HOT/WARM into COLD. * A background thread exists which shuffles items between/within the LRU's as limits are reached. This includes moves from COLD to WARM. * The background thread can also control the lru_crawler, if enabled. The primary goal is to better protect active items from "scanning". Items which are never hit again will flow from HOT, through COLD, and out the bottom. Items occasionally active (reaching COLD, but being hit before eviction), move to WARM. There they can stay relatively protected. A secondary goal is to improve latency. The LRU locks are no longer used on most item reads, largely during sets and from the background thread. Also the background thread is likely to find expired items and release them back to the slab class asynchronously, which speeds up new allocations. It is recommended to use this feature with the lru crawler as well: `memcached -o lru_maintainer,lru_crawler` - Then it will automatically scan slab classes for items with expired TTL's. If your items are always set to never expire, you can omit this option safely. An extra option: `-o temporary_ttl=N` (when used with lru_maintainer) will make items with a TTL less than or equal to this value use a fourth TEMP LRU. Items stored in TEMP are never bumped within its LRU or moved to other LRU's. They also cannot be evicted. This can help reduce holes and load on the LRU crawler. Do not set temporary_ttl too high or memory could become exhausted. memcached-1.5.22/doc/tls.txt0000664000175000017500000001675213575022500012554 00000000000000Securing Memcached with TLS Requirements ------------ We are required to encrypt Memcached network traffic as we deploy our servers in public cloud environments. We decided to implement SSL/TLS for TCP at the network layer of Memcached using OpenSSL libraries. This provides following benefits with the expense of added latency and reduced throughput (to be quantified). # Encryption :Data is encrypted on the wire between Memcached client and server. # Authentication : Optionally, both server and client authenticate each other. # Integrity: Data is not tampered or altered when transmitted between client and server Following are a few additional features. # Certificate refresh: when the server gets a new certificate, new connections will use new certificates without a need of re-starting the server process. # Multiple ports with and without TLS : by default all TCP ports are secured. Optionally we can setup the server to secure a specific TCP port. Note that initial implementation does not support session resumption or renegotiation. Design ------ We experimented two options for implementing TLS, with SSL buffered events and directly using OpenSSL API. Bufferevents can use the OpenSSL library to implement SSL/TLS. Our experiment used a socket-based bufferevent that tells OpenSSL to communicate with the network directly over. Unlike a worker thread sets callback on the socket, this uses a “bufferevent” object for callbacks. Memcached still has to setup the SSL Context but SSL handshake and object management is done via the “bufferevent_” API. While this was fairly easy to implement, we noticed a higher memory usage as we don’t have much control over allocating evbuffer objects in bufferevents. More over there is a discussion on removing the libevent dependency from Memcached; hence this option was not chosen. OpenSSL library provides APIs for us to directly read/write from a socket. With this option, we create an SSL Context and many SSL objects. The SSL Context object, created at the process level, holds certificates, a private key, and options regarding the TLS protocol and algorithms. SSL objects, created at the connection level, represents SSL sessions. SSL objects are responsible for encryption, and session handshake among other things. There are two ways to do network IO over TLS, either only use SSL_read/SSL_write with a network socket or use the API along with an output/input buffer pair. These buffers are referred as BIO (Basic Input Output) buffers. We started with the first option, create SSL objects with the socket and only interact with SSL_read/SSL_write. +------+ +-----+ |......|--> read(fd) --> BIO_write(rbio) -->|.....|--> SSL_read(ssl) --> IN |......| |.....| |.sock.| |.SSL.| |......| |.....| |......|<-- write(fd) <-- BIO_read(wbio) <--|.....|<-- SSL_write(ssl) <-- OUT +------+ +-----+ | | | | |<-------------------------------->| |<------------------->| | encrypted bytes | | unencrypted bytes | Figure 1 : Network sockets, BIO buffers and SSL_read/SSL_write (reference: https://gist.github.com/darrenjs/4645f115d10aa4b5cebf57483ec82eca) Memcached uses non blocking sockets and implements a rather complex state machine for network IO. A listener thread does the TCP handshake and initiates the SSL handshake after creating an SSL object based on the SSL Context object of the server. If there are no fatal errors, the listener thread hands over the socket to a worker thread. A worker completes the SSL handshake. ----------- ---------------------- | | Client | | Memcached Server | | | |--------------------- | | Listener thread | | TCP connect | | |---------------------> | (accept) | | ClientHello | | |---------------------> | (SSL_accept) | | | | | ServerHello and | | | Certificate, | | | ServerHelloDone | | | <---------------------| | | |--------------------- | | | | | V | |------------------- | | Worker thread | | ClientKeyExchange, | | | ChangeCipherSpec, | | | Finished | | |---------------------> | (SSL_read) | | | | | | | | NewSessionTicket, | | | ChangeCipherSpec, | | | Finished | | | <---------------------| | | | | | Memcached request/ | | | response | | | <-------------------> | (SSL_read/ | | | SSL_write) | ----------- ------------------------- Figure 2 : The initial SSL handshake Setting-up callbacks when the socket is ready for reading/writing is the same for both TLS and non-TLS connections. When the socket is ready, the state machine kicks off and issues a SSL_read/ SSL_write. Note that we implement a SSL_sendmsg wrapper on top of SSL_write to simulate the sendmsg API. This way we don't explicitly use BIO buffers or do BIO_write/BIO_read, but let OpenSSL library to do it on our behalf. Existing state machine takes care of reading the correct amount of bytes and do the error handling when needed. As a best practice, server certificates and keys are periodically refreshed by the PKI. When this happens we want server to use the new certificate without restarting the process. Memcached is a cache and restarting servers affects the latency of applications. We implement the automatic certificate refresh through a command. Upon receiving the "refresh_certs" command, the server reloads the certificates and key to the SSL Context object. Existing connection won't be interrupted but new connections will use the new certificate. We understand not all users want to use TLS or have the OpenSSL dependency. Therefore it's an optional module at the compile time. We can build a TLS capable Memcached server with "./configure --enable-tls". Once the server is built with TLS support, we can enabled it with "-Z" flag or "--enable-ssl". Certificate (-o ssl_chain_cert) and (-o ssl_key) are required parameters while others are optional. Supported options can be listed through "memcached -h". Developers need to have libio-socket-ssl-perl installed for running unit tests. When the server is built with TLS support, we can use "test_tls" make target to run all existing tests over TLS and some additional TLS specific tests. The minimum required OpenSSL version is 1.1.0g. memcached-1.5.22/doc/protocol.txt0000664000175000017500000021026213615376577013630 00000000000000Protocol -------- Clients of memcached communicate with server through TCP connections. (A UDP interface is also available; details are below under "UDP protocol.") A given running memcached server listens on some (configurable) port; clients connect to that port, send commands to the server, read responses, and eventually close the connection. There is no need to send any command to end the session. A client may just close the connection at any moment it no longer needs it. Note, however, that clients are encouraged to cache their connections rather than reopen them every time they need to store or retrieve data. This is because memcached is especially designed to work very efficiently with a very large number (many hundreds, more than a thousand if necessary) of open connections. Caching connections will eliminate the overhead associated with establishing a TCP connection (the overhead of preparing for a new connection on the server side is insignificant compared to this). There are two kinds of data sent in the memcache protocol: text lines and unstructured data. Text lines are used for commands from clients and responses from servers. Unstructured data is sent when a client wants to store or retrieve data. The server will transmit back unstructured data in exactly the same way it received it, as a byte stream. The server doesn't care about byte order issues in unstructured data and isn't aware of them. There are no limitations on characters that may appear in unstructured data; however, the reader of such data (either a client or a server) will always know, from a preceding text line, the exact length of the data block being transmitted. Text lines are always terminated by \r\n. Unstructured data is _also_ terminated by \r\n, even though \r, \n or any other 8-bit characters may also appear inside the data. Therefore, when a client retrieves data from a server, it must use the length of the data block (which it will be provided with) to determine where the data block ends, and not the fact that \r\n follows the end of the data block, even though it does. Keys ---- Data stored by memcached is identified with the help of a key. A key is a text string which should uniquely identify the data for clients that are interested in storing and retrieving it. Currently the length limit of a key is set at 250 characters (of course, normally clients wouldn't need to use such long keys); the key must not include control characters or whitespace. Commands -------- There are three types of commands. Storage commands (there are six: "set", "add", "replace", "append" "prepend" and "cas") ask the server to store some data identified by a key. The client sends a command line, and then a data block; after that the client expects one line of response, which will indicate success or failure. Retrieval commands ("get", "gets", "gat", and "gats") ask the server to retrieve data corresponding to a set of keys (one or more keys in one request). The client sends a command line, which includes all the requested keys; after that for each item the server finds it sends to the client one response line with information about the item, and one data block with the item's data; this continues until the server finished with the "END" response line. All other commands don't involve unstructured data. In all of them, the client sends one command line, and expects (depending on the command) either one line of response, or several lines of response ending with "END" on the last line. A command line always starts with the name of the command, followed by parameters (if any) delimited by whitespace. Command names are lower-case and are case-sensitive. Meta Commands [EXPERIMENTAL: MAY CHANGE] ------------- Meta commands are a set of new ASCII-based commands. They follow the same structure of the basic commands but have a new flexible feature set. Meta commands incorporate most features available in the binary protocol, as well as a flag system to make the commands flexible rather than having a large number of high level commands. These commands completely replace the usage of basic Storage and Retrieval commands. Meta commands are EXPERIMENTAL and may change syntax as of this writing. They are documented below the basic commands. These work mixed with normal protocol commands on the same connection. All existing commands continue to work. The meta commands will not replace specialized commands that do not sit in the Storage or Retrieval categories noted in the 'Commands' section above. All meta commands follow a basic syntax: cm <...>\r\n Where 'cm' is a 2 character command code. The number of tokens supplied is based off of the flags used. Responses look like: RC <...>\r\n Where 'rc' is a 2 character return code. The number of tokens returned are based off of the flags used. Flags are single character codes, ie 'q' or 'k' or 'I', which adjust the behavior of the command. The flags are reflected in the response. The order of which tokens are consumed or returned depend on the order of the flags given. For example, a metaget with flags of 'st' would return tokens for "size" and "TTL remaining" in that order. 'ts' would return "TTL remaining" then "size". Syntax errors are handled the same as noted under 'Error strings' section below. For usage examples beyond basic syntax, please see the wiki: https://github.com/memcached/memcached/wiki/MetaCommands Expiration times ---------------- Some commands involve a client sending some kind of expiration time (relative to an item or to an operation requested by the client) to the server. In all such cases, the actual value sent may either be Unix time (number of seconds since January 1, 1970, as a 32-bit value), or a number of seconds starting from current time. In the latter case, this number of seconds may not exceed 60*60*24*30 (number of seconds in 30 days); if the number sent by a client is larger than that, the server will consider it to be real Unix time value rather than an offset from current time. Note that a TTL of 1 will sometimes immediately expire. Time is internally updated on second boundaries, which makes expiration time roughly +/- 1s. This more proportionally affects very low TTL's. Error strings ------------- Each command sent by a client may be answered with an error string from the server. These error strings come in three types: - "ERROR\r\n" means the client sent a nonexistent command name. - "CLIENT_ERROR \r\n" means some sort of client error in the input line, i.e. the input doesn't conform to the protocol in some way. is a human-readable error string. - "SERVER_ERROR \r\n" means some sort of server error prevents the server from carrying out the command. is a human-readable error string. In cases of severe server errors, which make it impossible to continue serving the client (this shouldn't normally happen), the server will close the connection after sending the error line. This is the only case in which the server closes a connection to a client. In the descriptions of individual commands below, these error lines are not again specifically mentioned, but clients must allow for their possibility. Authentication -------------- Optional username/password token authentication (see -Y option). Used by sending a fake "set" command with any key: set \r\n username password\r\n key, flags, and exptime are ignored for authentication. Bytes is the length of the username/password payload. - "STORED\r\n" indicates success. After this point any command should work normally. - "CLIENT_ERROR [message]\r\n" will be returned if authentication fails for any reason. Storage commands ---------------- First, the client sends a command line which looks like this: [noreply]\r\n cas [noreply]\r\n - is "set", "add", "replace", "append" or "prepend" "set" means "store this data". "add" means "store this data, but only if the server *doesn't* already hold data for this key". "replace" means "store this data, but only if the server *does* already hold data for this key". "append" means "add this data to an existing key after existing data". "prepend" means "add this data to an existing key before existing data". The append and prepend commands do not accept flags or exptime. They update existing data portions, and ignore new flag and exptime settings. "cas" is a check and set operation which means "store this data but only if no one else has updated since I last fetched it." - is the key under which the client asks to store the data - is an arbitrary 16-bit unsigned integer (written out in decimal) that the server stores along with the data and sends back when the item is retrieved. Clients may use this as a bit field to store data-specific information; this field is opaque to the server. Note that in memcached 1.2.1 and higher, flags may be 32-bits, instead of 16, but you might want to restrict yourself to 16 bits for compatibility with older versions. - is expiration time. If it's 0, the item never expires (although it may be deleted from the cache to make place for other items). If it's non-zero (either Unix time or offset in seconds from current time), it is guaranteed that clients will not be able to retrieve this item after the expiration time arrives (measured by server time). If a negative value is given the item is immediately expired. - is the number of bytes in the data block to follow, *not* including the delimiting \r\n. may be zero (in which case it's followed by an empty data block). - is a unique 64-bit value of an existing entry. Clients should use the value returned from the "gets" command when issuing "cas" updates. - "noreply" optional parameter instructs the server to not send the reply. NOTE: if the request line is malformed, the server can't parse "noreply" option reliably. In this case it may send the error to the client, and not reading it on the client side will break things. Client should construct only valid requests. After this line, the client sends the data block: \r\n - is a chunk of arbitrary 8-bit data of length from the previous line. After sending the command line and the data block the client awaits the reply, which may be: - "STORED\r\n", to indicate success. - "NOT_STORED\r\n" to indicate the data was not stored, but not because of an error. This normally means that the condition for an "add" or a "replace" command wasn't met. - "EXISTS\r\n" to indicate that the item you are trying to store with a "cas" command has been modified since you last fetched it. - "NOT_FOUND\r\n" to indicate that the item you are trying to store with a "cas" command did not exist. Retrieval command: ------------------ The retrieval commands "get" and "gets" operate like this: get *\r\n gets *\r\n - * means one or more key strings separated by whitespace. After this command, the client expects zero or more items, each of which is received as a text line followed by a data block. After all the items have been transmitted, the server sends the string "END\r\n" to indicate the end of response. Each item sent by the server looks like this: VALUE []\r\n \r\n - is the key for the item being sent - is the flags value set by the storage command - is the length of the data block to follow, *not* including its delimiting \r\n - is a unique 64-bit integer that uniquely identifies this specific item. - is the data for this item. If some of the keys appearing in a retrieval request are not sent back by the server in the item list this means that the server does not hold items with such keys (because they were never stored, or stored but deleted to make space for more items, or expired, or explicitly deleted by a client). Deletion -------- The command "delete" allows for explicit deletion of items: delete [noreply]\r\n - is the key of the item the client wishes the server to delete - "noreply" optional parameter instructs the server to not send the reply. See the note in Storage commands regarding malformed requests. The response line to this command can be one of: - "DELETED\r\n" to indicate success - "NOT_FOUND\r\n" to indicate that the item with this key was not found. See the "flush_all" command below for immediate invalidation of all existing items. Increment/Decrement ------------------- Commands "incr" and "decr" are used to change data for some item in-place, incrementing or decrementing it. The data for the item is treated as decimal representation of a 64-bit unsigned integer. If the current data value does not conform to such a representation, the incr/decr commands return an error (memcached <= 1.2.6 treated the bogus value as if it were 0, leading to confusion). Also, the item must already exist for incr/decr to work; these commands won't pretend that a non-existent key exists with value 0; instead, they will fail. The client sends the command line: incr [noreply]\r\n or decr [noreply]\r\n - is the key of the item the client wishes to change - is the amount by which the client wants to increase/decrease the item. It is a decimal representation of a 64-bit unsigned integer. - "noreply" optional parameter instructs the server to not send the reply. See the note in Storage commands regarding malformed requests. The response will be one of: - "NOT_FOUND\r\n" to indicate the item with this value was not found - \r\n , where is the new value of the item's data, after the increment/decrement operation was carried out. Note that underflow in the "decr" command is caught: if a client tries to decrease the value below 0, the new value will be 0. Overflow in the "incr" command will wrap around the 64 bit mark. Note also that decrementing a number such that it loses length isn't guaranteed to decrement its returned length. The number MAY be space-padded at the end, but this is purely an implementation optimization, so you also shouldn't rely on that. Touch ----- The "touch" command is used to update the expiration time of an existing item without fetching it. touch [noreply]\r\n - is the key of the item the client wishes the server to touch - is expiration time. Works the same as with the update commands (set/add/etc). This replaces the existing expiration time. If an existing item were to expire in 10 seconds, but then was touched with an expiration time of "20", the item would then expire in 20 seconds. - "noreply" optional parameter instructs the server to not send the reply. See the note in Storage commands regarding malformed requests. The response line to this command can be one of: - "TOUCHED\r\n" to indicate success - "NOT_FOUND\r\n" to indicate that the item with this key was not found. Get And Touch ------------- The "gat" and "gats" commands are used to fetch items and update the expiration time of an existing items. gat *\r\n gats *\r\n - is expiration time. - * means one or more key strings separated by whitespace. After this command, the client expects zero or more items, each of which is received as a text line followed by a data block. After all the items have been transmitted, the server sends the string "END\r\n" to indicate the end of response. Each item sent by the server looks like this: VALUE []\r\n \r\n - is the key for the item being sent - is the flags value set by the storage command - is the length of the data block to follow, *not* including its delimiting \r\n - is a unique 64-bit integer that uniquely identifies this specific item. - is the data for this item. Meta Debug ---------- The meta debug command is a human readable dump of all available internal metadata of an item, minus the value. me \r\n - means one key string. The response looks like: ME =*\r\nEN\r\n Each of the keys and values are the internal data for the item. exp = expiration time la = time in seconds since last access cas = CAS ID fetch = whether an item has been fetched before cls = slab class id size = total size in bytes Others may be added. Meta Get -------- The meta get command is the generic command for retrieving key data from memcached. Based on the flags supplied, it can replace all of the commands: "get", "gets", "gat", "gats", "touch", as well as adding new options. mg *\r\n - means one key string. Unlike "get" metaget can only take a single key. - are a set of single character codes ended with a space or newline. - * means zero or more tokens, based on the flags supplied. They are consumed in order specified by the flags. After this command, the client expects an item to be returned, received as a text line followed by an optional data block. After the item is transferred, the server sends the string "EN\r\n" to indicate the end of the response. If a response line appearing in a retrieval request is not sent back by the server this means that the server does not have the item (because it was never stored, or stored but deleted to make space for more items, or expired, or explicitly deleted by a client). An item sent by the server looks like: VA *\r\n \r\n - are a reflection of the flags sent to the server. Extra flags can be added to the response based on the execution of the command. - * are tokens returned by the server, based on the flags supplied. They are added in order specified by the flags sent. - is the data for this item. Note that the data black is optional, requiring the 'v' flag to be supplied. The flags used by the 'mg' command are: - c: return item cas token - f: return client flags token - h: return whether item has been hit before as a 0 or 1 - k: return key as a token - l: return time since item was last accessed in seconds - O (token): opaque value, consumes a token and copies back with response - q: use noreply semantics for return codes. - s: return item size token - t: return item TTL remaining in seconds (-1 for unlimited) - u: don't bump the item in the LRU - v: return item value in These flags can modify the item: - N (token): vivify on miss, takes TTL as a argument - R (token): if token is less than remaining TTL win for recache - T (token): update remaining TTL These extra flags can be added to the response: - W: client has "won" the recache flag - X: item is stale - Z: tem has already sent a winning flag The flags are now repeated with detailed information where useful: - h: return whether item has been hit before as a 0 or 1 - l: return time since item was last accessed in seconds The above two flags return the value of "hit before?" and "last access time" before the command was processed. Otherwise this would always show a 1 for hit or always show an access time of "0" unless combined with the "u" flag. - O (token): opaque value, consumes a token and copies back with response The O (opaque) token is used by this and other commands to allow easier pipelining of requests while saving bytes on the wire for responses. For example: if pipelining three get commands together, you may not know which response belongs to which without also retrieving the key. If the key is very long this can generate a lot of traffic, especially if the data block is very small. Instead, you can supply an "O" flag for each mg with tokens of "1" "2" and "3", to match up responses to the request. Opaque tokens may be up to 32 bytes in length, and are a string similar to keys. - q: use noreply semantics for return codes. Noreply is a method of reducing the amount of data sent back by memcached to the client for normal responses. In the case of metaget, if an item is not available the "VA" line is normally not sent, and the response is terminated by "EN\r\n". With noreply enabled, the "EN\r\n" marker is suppressed. This allows you to pipeline several mg's together and read all of the responses without the "EN\r\n" lines in the middle. Errors are always returned. - u: don't bump the item in the LRU It is possible to access an item without causing it to be "bumped" to the head of the LRU. This also avoids marking an item as being hit or updating its last access time. - v: return item value in The data block for a metaget response is optional, requiring this flag to be passed in. These flags can modify the item: - N (token): vivify on miss, takes TTL as a argument Used to help with so called "dog piling" problems with recaching of popular items. If supplied, and metaget does not find the item in cache, it will create a stub item with the key and TTL as supplied. If such an item is created a 'W' flag is added to the response to indicate to a client that they have "won" the right to recache an item. The automatically created item has 0 bytes of data. Further requests will see a 'Z' flag to indicate that another client has already received the win flag. Can be combined with CAS flags to gate the update further. - R (token): if token is less than remaining TTL win for recache Similar to and can be combined with 'N'. If the remaining TTL of an item is below the supplied token, return a 'W' flag to indicate the client has "won" the right to recache an item. This allows refreshing an item before it leads to a miss. - T (token): update remaining TTL Similar to "touch" and "gat" commands, updates the remaining TTL of an item if hit. These extra flags can be added to the response: - W: client has "won" the recache flag When combined with N or R flags, a client may be informed they have "won" ownership of a cache item. This allows a single client to be atomically notified that it should cache or update an item. Further clients requesting the item can use the existing data block (if valid or stale), retry, wait, or take some other action while the item is missing. This is used when the act of recaching an item can cause undue load on another system (CPU, database accesses, time, and so on). - X: item is stale Items can be marked as stale by the metadelete command. This indicates to the client that an object needs to be updated, and that it should make a decision o if potentially stale data is safe to use. This is used when you want to convince clients to recache an item, but it's safe to use pre-existing data while the recache happens. - Z: item has already sent a winning flag When combined with the X flag, or the client N or R flags, this extra response flag indicates to a client that a different client is responsible for recaching this item. If there is data supplied it may use it, or the client may decide to retry later or take some other action. Meta Set -------- The meta set command a generic command for storing data to memcached. Based on the flags supplied, it can replace the commands: "set", "cas". as well as adding new options. ms *\r\n - means one key string. - are a set of single character codes ended with a space or newline. - * means zero or more tokens, based on the flags supplied. They are consumed in order specified by the flags. After this line, the client sends the data block: \r\n - is a chunk of arbitrary 8-bit data of length supplied by an 'S' flag and token from the previous line. If no 'S' flag is supplied the data is assumed to be 0 length. After sending the command line and the data block the client awaits the reply, which is of the format: CD *\r\n Where CD is one of: - "ST" (STORED), to indicate success. - "NS" (NOT_STORED), to indicate the data was not stored, but not because of an error. - "EX" (EXISTS), to indicate that the item you are trying to store with CAS semantics has been modified since you last fetched it. - "NF" (NOT_FOUND), to indicate that the item you are trying to store with CAS semantics did not exist. The flags used by the 'ms' command are: - C (token): compare CAS value when storing item - F (token): set client flags to token (32 bit unsigned numeric) - I: invalid. set-to-invalid if CAS is older than it should be. - k: return key as a token - O (token): opaque value, consumes a token and copies back with response - q: use noreply semantics for return codes - S (token): size of to store - T (token): Time-To-Live for item, see "Expiration" above. The flags are now repeated with detailed information where useful: - C (token): compare CAS value when storing item Similar to the basic "cas" command, only store item if the supplied token matches the current CAS value of the item. When combined with the 'I' flag, a CAS value that is _lower_ than the current value may be accepted, but the item will be marked as "stale", returning the X flag with mget requests. - F (token): set client flags to token (32 bit unsigned numeric) Sets flags to 0 if not supplied. - I: invalid. set-to-invalid if CAS is older than it should be. Functional when combined with 'C' flag above. - O (token): opaque value, consumes a token and copies back with response See description under 'Meta Get' - q: use noreply semantics for return codes Noreply is a method of reducing the amount of data sent back by memcached to the client for normal responses. In the case of metaset, a response that would start with "ST" (STORED) will not be sent. Any other code, such as "EX" (EXISTS) will still be returned. Errors are always returned. Meta Delete ----------- The meta delete command allows for explicit deletion of items, as well as marking items as "stale" to allow serving items as stale during revalidation. md *\r\n - means one key string. - are a set of single character codes ended with a space or newline. - * means zero or more tokens, based on the flags supplied. They are consumed in order specified by the flags. The response is in the format: CD *\r\n Where CD is one of: - "DE" (DELETED), to indicate success - "NF" (NOT_FOUND), to indicate that the item with this key was not found. - "EX" (EXISTS), to indicate that the supplied CAS token does not match the stored item. The flags used by the 'md' command are: - C (token): compare CAS value - I: invalidate. mark as stale, bumps CAS. - k: return key - O: opaque to copy back. - q: noreply - T (token): updates TTL, only when paired with the 'I' flag The flags are now repeated with detailed information where useful: - C (token): compare CAS value Can be used to only delete or mark-stale if a supplied CAS value matches. - I: invalidate. mark as stale, bumps CAS. Instead of removing an item, this will give the item a new CAS value and mark it as stale. This means when it is later fetched by metaget, the client will be supplied an 'X' flag to show the data is stale and needs to be recached. - O (token): opaque to copy back. See description under 'Meta Get' - q: noreply See description under 'Meta Set'. In the case of meta delete, this will hide response lines with the code "DE". It will still return any other responses. - T (token): updates TTL, only when paired with the 'I' flag When marking an item as stale with 'I', the 'T' flag can be used to update the TTL as well; limiting the amount of time an item will live while stale and waiting to be recached. Meta No-Op ---------- The meta no-op command exists solely to return a static response code. It takes no flags, no arguments. This returns the static response: "EN\r\n" Similar to a meta get command. This command is useful when used with the 'q' flag and pipelining commands. For example, with 'mg' the response lines are blank on miss, and the 'q' flag will hide the "EN\r\n" response code. If pipelining several 'mg's together with noreply semantics, an "mn\r\n" command can be tagged to the end of the chain, which will return an "EN\r\n", signalling to a client that all previous commands have been processed. Slabs Reassign -------------- NOTE: This command is subject to change as of this writing. The slabs reassign command is used to redistribute memory once a running instance has hit its limit. It might be desirable to have memory laid out differently than was automatically assigned after the server started. slabs reassign \r\n - is an id number for the slab class to steal a page from A source class id of -1 means "pick from any valid class" - is an id number for the slab class to move a page to The response line could be one of: - "OK" to indicate the page has been scheduled to move - "BUSY [message]" to indicate a page is already being processed, try again later. - "BADCLASS [message]" a bad class id was specified - "NOSPARE [message]" source class has no spare pages - "NOTFULL [message]" dest class must be full to move new pages to it - "UNSAFE [message]" source class cannot move a page right now - "SAME [message]" must specify different source/dest ids. Slabs Automove -------------- NOTE: This command is subject to change as of this writing. The slabs automove command enables a background thread which decides on its own when to move memory between slab classes. Its implementation and options will likely be in flux for several versions. See the wiki/mailing list for more details. The automover can be enabled or disabled at runtime with this command. slabs automove <0|1> - 0|1|2 is the indicator on whether to enable the slabs automover or not. The response should always be "OK\r\n" - <0> means to set the thread on standby - <1> means to return pages to a global pool when there are more than 2 pages worth of free chunks in a slab class. Pages are then re-assigned back into other classes as-needed. - <2> is a highly aggressive mode which causes pages to be moved every time there is an eviction. It is not recommended to run for very long in this mode unless your access patterns are very well understood. LRU Tuning ---------- Memcached supports multiple LRU algorithms, with a few tunables. Effort is made to have sane defaults however you are able to tune while the daemon is running. The traditional model is "flat" mode, which is a single LRU chain per slab class. The newer (with `-o modern` or `-o lru_maintainer`) is segmented into HOT, WARM, COLD. There is also a TEMP LRU. See doc/new_lru.txt for details. lru