pylibmc-1.2.3/0000755000076500000240000000000011746271113014120 5ustar lericsonstaff00000000000000pylibmc-1.2.3/_pylibmcmodule.c0000644000076500000240000020016611746271101017272 0ustar lericsonstaff00000000000000/** * _pylibmc: hand-made libmemcached bindings for Python * * Copyright (c) 2008, Ludvig Ericson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * - Neither the name of the author nor the names of the contributors may be * used to endorse or promote products derived from this software without * specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "_pylibmcmodule.h" #ifdef USE_ZLIB # include # define ZLIB_BUFSZ (1 << 14) /* only callable while holding the GIL */ # define _ZLIB_ERR(s, rc) \ PyErr_Format(PylibMCExc_MemcachedError, "zlib error %d in " s, rc); #endif #define PyBool_TEST(t) ((t) ? Py_True : Py_False) #define PyModule_ADD_REF(mod, nam, obj) \ { Py_INCREF(obj); \ PyModule_AddObject(mod, nam, obj); } /* {{{ Type methods */ static PylibMC_Client *PylibMC_ClientType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PylibMC_Client *self; /* GenericNew calls GenericAlloc (via the indirection type->tp_alloc) which * adds GC tracking if flagged for, and also calls PyObject_Init. */ self = (PylibMC_Client *)PyType_GenericNew(type, args, kwds); if (self != NULL) { self->mc = memcached_create(NULL); self->sasl_set = false; } return self; } static void PylibMC_ClientType_dealloc(PylibMC_Client *self) { if (self->mc != NULL) { #if LIBMEMCACHED_WITH_SASL_SUPPORT if (self->sasl_set) { memcached_destroy_sasl_auth_data(self->mc); } #endif memcached_free(self->mc); } self->ob_type->tp_free(self); } /* }}} */ static int PylibMC_Client_init(PylibMC_Client *self, PyObject *args, PyObject *kwds) { PyObject *srvs, *srvs_it, *c_srv; unsigned char set_stype = 0, bin = 0, got_server = 0; const char *user = NULL, *pass = NULL; memcached_return rc; static char *kws[] = { "servers", "binary", "username", "password", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|bzz", kws, &srvs, &bin, &user, &pass)) { return -1; } if ((srvs_it = PyObject_GetIter(srvs)) == NULL) { return -1; } /* setup sasl */ if (user != NULL || pass != NULL) { #if LIBMEMCACHED_WITH_SASL_SUPPORT if (user == NULL || pass == NULL) { PyErr_SetString(PyExc_TypeError, "SASL requires both username and password"); goto error; } if (!bin) { PyErr_SetString(PyExc_TypeError, "SASL requires the memcached binary protocol"); goto error; } rc = memcached_set_sasl_auth_data(self->mc, user, pass); if (rc != MEMCACHED_SUCCESS) { PylibMC_ErrFromMemcached(self, "memcached_set_sasl_auth_data", rc); goto error; } /* Can't just look at the memcached_st->sasl data, because then it * breaks in libmemcached 0.43 and potentially earlier. */ self->sasl_set = true; #else PyErr_SetString(PyExc_TypeError, "libmemcached does not support SASL"); goto error; #endif } rc = memcached_behavior_set(self->mc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, bin); if (rc != MEMCACHED_SUCCESS) { PyErr_SetString(PyExc_RuntimeError, "binary protocol behavior set failed"); goto error; } while ((c_srv = PyIter_Next(srvs_it)) != NULL) { unsigned char stype; char *hostname; unsigned short int port; got_server |= 1; port = 0; if (PyString_Check(c_srv)) { memcached_server_st *list; list = memcached_servers_parse(PyString_AS_STRING(c_srv)); if (list == NULL) { PyErr_SetString(PylibMCExc_MemcachedError, "memcached_servers_parse returned NULL"); goto it_error; } rc = memcached_server_push(self->mc, list); free(list); if (rc != MEMCACHED_SUCCESS) { PylibMC_ErrFromMemcached(self, "memcached_server_push", rc); goto it_error; } } else if (PyArg_ParseTuple(c_srv, "Bs|H", &stype, &hostname, &port)) { if (set_stype && set_stype != stype) { PyErr_SetString(PyExc_ValueError, "can't mix transport types"); goto it_error; } else { set_stype = stype; if (stype == PYLIBMC_SERVER_UDP) { rc = memcached_behavior_set(self->mc, MEMCACHED_BEHAVIOR_USE_UDP, 1); if (rc != MEMCACHED_SUCCESS) { PyErr_SetString(PyExc_RuntimeError, "udp behavior set failed"); goto it_error; } } } switch (stype) { case PYLIBMC_SERVER_TCP: rc = memcached_server_add(self->mc, hostname, port); break; case PYLIBMC_SERVER_UDP: rc = memcached_server_add_udp(self->mc, hostname, port); break; case PYLIBMC_SERVER_UNIX: if (port) { PyErr_SetString(PyExc_ValueError, "can't set port on unix sockets"); goto it_error; } rc = memcached_server_add_unix_socket(self->mc, hostname); break; default: PyErr_Format(PyExc_ValueError, "bad type: %u", stype); goto it_error; } if (rc != MEMCACHED_SUCCESS) { PylibMC_ErrFromMemcached(self, "memcached_server_add_*", rc); goto it_error; } } Py_DECREF(c_srv); continue; it_error: Py_DECREF(c_srv); goto error; } if (!got_server) { PyErr_SetString(PylibMCExc_MemcachedError, "empty server list"); goto error; } Py_DECREF(srvs_it); return 0; error: Py_DECREF(srvs_it); return -1; } /* {{{ Compression helpers */ #ifdef USE_ZLIB static int _PylibMC_Deflate(char *value, size_t value_len, char **result, size_t *result_len) { /* FIXME Failures are entirely silent. */ int rc; z_stream strm; *result = NULL; *result_len = 0; /* Don't ask me about this one. Got it from zlibmodule.c in Python 2.6. */ ssize_t out_sz = value_len + value_len / 1000 + 12 + 1; if ((*result = malloc(out_sz)) == NULL) { goto error; } /* TODO Should break up next_in into blocks of max 0xffffffff in length. */ assert(value_len < 0xffffffffU); assert(out_sz < 0xffffffffU); strm.avail_in = (uInt)value_len; strm.avail_out = (uInt)out_sz; strm.next_in = (Bytef *)value; strm.next_out = (Bytef *)*result; /* we just pre-allocated all of it up front */ strm.zalloc = (alloc_func)NULL; strm.zfree = (free_func)Z_NULL; /* TODO Expose compression level somehow. */ if (deflateInit((z_streamp)&strm, Z_BEST_SPEED) != Z_OK) { goto error; } Py_BEGIN_ALLOW_THREADS; rc = deflate((z_streamp)&strm, Z_FINISH); Py_END_ALLOW_THREADS; if (rc != Z_STREAM_END) { _ZLIB_ERR("deflate", rc); goto error; } if (deflateEnd((z_streamp)&strm) != Z_OK) { goto error; } if(strm.total_out >= value_len) { /* if we didn't actually save anything, don't bother storing it compressed */ goto error; } /* *result should already be populated since that's the address we passed into the z_stream */ *result_len = strm.total_out; return 1; error: /* if any error occurred, we'll just use the original value instead of trying to compress it */ if(*result != NULL) { free(*result); *result = NULL; } return 0; } static PyObject *_PylibMC_Inflate(char *value, size_t size) { int rc; char *out; PyObject *out_obj; Py_ssize_t rvalsz; z_stream strm; /* Output buffer */ rvalsz = ZLIB_BUFSZ; out_obj = PyString_FromStringAndSize(NULL, rvalsz); if (out_obj == NULL) { return NULL; } out = PyString_AS_STRING(out_obj); /* TODO 64-bit fix size/rvalsz */ assert(size < 0xffffffffU); assert(rvalsz < 0xffffffffU); /* Set up zlib stream. */ strm.avail_in = (uInt)size; strm.avail_out = (uInt)rvalsz; strm.next_in = (Byte *)value; strm.next_out = (Byte *)out; strm.zalloc = (alloc_func)NULL; strm.zfree = (free_func)Z_NULL; /* TODO Add controlling of windowBits with inflateInit2? */ if ((rc = inflateInit((z_streamp)&strm)) != Z_OK) { _ZLIB_ERR("inflateInit", rc); goto error; } do { Py_BEGIN_ALLOW_THREADS; rc = inflate((z_streamp)&strm, Z_FINISH); Py_END_ALLOW_THREADS; switch (rc) { case Z_STREAM_END: break; /* When a Z_BUF_ERROR occurs, we should be out of memory. * This is also true for Z_OK, hence the fall-through. */ case Z_BUF_ERROR: if (strm.avail_out) { _ZLIB_ERR("inflate", rc); inflateEnd(&strm); goto error; } /* Fall-through */ case Z_OK: if (_PyString_Resize(&out_obj, (Py_ssize_t)(rvalsz << 1)) < 0) { inflateEnd(&strm); goto error; } /* Wind forward */ out = PyString_AS_STRING(out_obj); strm.next_out = (Byte *)(out + rvalsz); strm.avail_out = (uInt)rvalsz; rvalsz = rvalsz << 1; break; default: inflateEnd(&strm); goto error; } } while (rc != Z_STREAM_END); if ((rc = inflateEnd(&strm)) != Z_OK) { _ZLIB_ERR("inflateEnd", rc); goto error; } _PyString_Resize(&out_obj, strm.total_out); return out_obj; error: Py_DECREF(out_obj); return NULL; } #endif /* }}} */ static PyObject *_PylibMC_parse_memcached_value(char *value, size_t size, uint32_t flags) { PyObject *retval = NULL; PyObject *tmp = NULL; uint32_t dtype = flags & PYLIBMC_FLAG_TYPES; #if USE_ZLIB PyObject *inflated = NULL; /* Decompress value if necessary. */ if (flags & PYLIBMC_FLAG_ZLIB) { if ((inflated = _PylibMC_Inflate(value, size)) == NULL) { return NULL; } value = PyString_AS_STRING(inflated); size = PyString_GET_SIZE(inflated); } #else if (flags & PYLIBMC_FLAG_ZLIB) { PyErr_SetString(PylibMCExc_MemcachedError, "value for key compressed, unable to inflate"); return NULL; } #endif switch (dtype) { case PYLIBMC_FLAG_PICKLE: retval = _PylibMC_Unpickle(value, size); break; case PYLIBMC_FLAG_INTEGER: case PYLIBMC_FLAG_LONG: case PYLIBMC_FLAG_BOOL: /* PyInt_FromString doesn't take a length param and we're not NULL-terminated, so we'll have to make an intermediate Python string out of it */ tmp = PyString_FromStringAndSize(value, size); if(tmp == NULL) { goto cleanup; } retval = PyInt_FromString(PyString_AS_STRING(tmp), NULL, 10); if(retval != NULL && dtype == PYLIBMC_FLAG_BOOL) { Py_DECREF(tmp); tmp = retval; retval = PyBool_FromLong(PyInt_AS_LONG(tmp)); } break; case PYLIBMC_FLAG_NONE: retval = PyString_FromStringAndSize(value, (Py_ssize_t)size); break; default: PyErr_Format(PylibMCExc_MemcachedError, "unknown memcached key flags %u", flags); } cleanup: #if USE_ZLIB Py_XDECREF(inflated); #endif Py_XDECREF(tmp); return retval; } static PyObject *_PylibMC_parse_memcached_result(memcached_result_st *res) { return _PylibMC_parse_memcached_value((char *)memcached_result_value(res), memcached_result_length(res), memcached_result_flags(res)); } static PyObject *PylibMC_Client_get(PylibMC_Client *self, PyObject *arg) { char *mc_val; size_t val_size; uint32_t flags; memcached_return error; if (!_PylibMC_CheckKey(arg)) { return NULL; } else if (!PySequence_Length(arg) ) { /* Others do this, so... */ Py_RETURN_NONE; } Py_BEGIN_ALLOW_THREADS; mc_val = memcached_get(self->mc, PyString_AS_STRING(arg), PyString_GET_SIZE(arg), &val_size, &flags, &error); Py_END_ALLOW_THREADS; if (mc_val != NULL) { PyObject *r = _PylibMC_parse_memcached_value(mc_val, val_size, flags); free(mc_val); return r; } else if (error == MEMCACHED_SUCCESS) { /* This happens for empty values, and so we fake an empty string. */ return PyString_FromStringAndSize("", 0); } else if (error == MEMCACHED_NOTFOUND) { /* Since python-memcache returns None when the key doesn't exist, * so shall we. */ Py_RETURN_NONE; } return PylibMC_ErrFromMemcachedWithKey(self, "memcached_get", error, PyString_AS_STRING(arg), PyString_GET_SIZE(arg)); } static PyObject *PylibMC_Client_gets(PylibMC_Client *self, PyObject *arg) { const char* keys[2]; size_t keylengths[2]; memcached_result_st *res = NULL; memcached_return rc; PyObject* ret = NULL; if (!_PylibMC_CheckKey(arg)) { return NULL; } else if (!PySequence_Length(arg)) { /* Others do this, so... */ Py_RETURN_NONE; } else if (!memcached_behavior_get(self->mc, MEMCACHED_BEHAVIOR_SUPPORT_CAS)) { PyErr_SetString(PyExc_ValueError, "gets without cas behavior"); return NULL; } /* Use an mget to fetch the key. * mget is the only function that returns a memcached_result_st, * which is the only way to get at the returned cas value. */ *keys = PyString_AS_STRING(arg); *keylengths = (size_t)PyString_GET_SIZE(arg); Py_BEGIN_ALLOW_THREADS; rc = memcached_mget(self->mc, keys, keylengths, 1); if (rc == MEMCACHED_SUCCESS) res = memcached_fetch_result(self->mc, res, &rc); Py_END_ALLOW_THREADS; if (rc == MEMCACHED_SUCCESS && res != NULL) { ret = Py_BuildValue("(NL)", _PylibMC_parse_memcached_result(res), memcached_result_cas(res)); /* we have to fetch the last result from the mget cursor */ if (NULL != memcached_fetch_result(self->mc, NULL, &rc)) { memcached_quit(self->mc); Py_DECREF(ret); ret = NULL; PyErr_SetString(PyExc_RuntimeError, "fetch not done"); } } else if (rc == MEMCACHED_END) { /* Key not found => (None, None) */ ret = Py_BuildValue("(OO)", Py_None, Py_None); } else { ret = PylibMC_ErrFromMemcached(self, "memcached_gets", rc); } if (res != NULL) { memcached_result_free(res); } return ret; } /* {{{ Set commands (set, replace, add, prepend, append) */ static PyObject *_PylibMC_RunSetCommandSingle(PylibMC_Client *self, _PylibMC_SetCommand f, char *fname, PyObject *args, PyObject *kwds) { /* function called by the set/add/etc commands */ static char *kws[] = { "key", "val", "time", "min_compress_len", NULL }; PyObject *key; PyObject *value; unsigned int time = 0; /* this will be turned into a time_t */ unsigned int min_compress = 0; bool success = false; if (!PyArg_ParseTupleAndKeywords(args, kwds, "SO|II", kws, &key, &value, &time, &min_compress)) { return NULL; } #ifndef USE_ZLIB if (min_compress) { PyErr_SetString(PyExc_TypeError, "min_compress_len without zlib"); return NULL; } #endif pylibmc_mset serialized = { NULL }; success = _PylibMC_SerializeValue(key, NULL, value, time, &serialized); if (!success) goto cleanup; success = _PylibMC_RunSetCommand(self, f, fname, &serialized, 1, min_compress); cleanup: _PylibMC_FreeMset(&serialized); if(PyErr_Occurred() != NULL) { return NULL; } else if(success) { Py_RETURN_TRUE; } else { Py_RETURN_FALSE; } } static PyObject *_PylibMC_RunSetCommandMulti(PylibMC_Client* self, _PylibMC_SetCommand f, char *fname, PyObject* args, PyObject* kwds) { /* function called by the set/add/incr/etc commands */ PyObject* keys = NULL; PyObject* key_prefix = NULL; unsigned int time = 0; unsigned int min_compress = 0; PyObject * retval = NULL; size_t idx = 0; static char *kws[] = { "keys", "time", "key_prefix", "min_compress_len", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|ISI", kws, &PyDict_Type, &keys, &time, &key_prefix, &min_compress)) { return NULL; } #ifndef USE_ZLIB if (min_compress) { PyErr_SetString(PyExc_TypeError, "min_compress_len without zlib"); return NULL; } #endif PyObject *curr_key, *curr_value; size_t nkeys = (size_t)PyDict_Size(keys); pylibmc_mset* serialized = PyMem_New(pylibmc_mset, nkeys); if (serialized == NULL) { goto cleanup; } /** * We're pointing into existing Python memory with the 'key' members of * pylibmc_mset (extracted using PyDict_Next) and during * _PylibMC_RunSetCommand (which uses those same 'key' params, and * potentially points into value string objects too), so we don't want to * go around decrementing any references that risk destroying the pointed * objects until we're done, especially since we're going to release the * GIL while we do the I/O that accesses that memory. We're assuming that * this is safe because Python strings are immutable */ Py_ssize_t pos = 0; /* PyDict_Next's 'pos' isn't an incrementing index */ for (idx = 0; PyDict_Next(keys, &pos, &curr_key, &curr_value); idx++) { int success = _PylibMC_SerializeValue(curr_key, key_prefix, curr_value, time, &serialized[idx]); if (!success || PyErr_Occurred() != NULL) { /* exception should already be on the stack */ goto cleanup; } } if (PyErr_Occurred() != NULL) { /* an iteration error of some sort */ goto cleanup; } bool allsuccess = _PylibMC_RunSetCommand(self, f, fname, serialized, nkeys, min_compress); if (PyErr_Occurred() != NULL) { goto cleanup; } /* Return value for set_multi, which is a list of keys which failed to be set */ if ((retval = PyList_New(0)) == NULL) return PyErr_NoMemory(); for (idx = 0; !allsuccess && idx < nkeys; idx++) { if (serialized[idx].success) continue; if (PyList_Append(retval, serialized[idx].key_obj) != 0) { /* Ugh */ Py_DECREF(retval); retval = PyErr_NoMemory(); goto cleanup; } } cleanup: if (serialized != NULL) { for (pos = 0; pos < nkeys; pos++) { _PylibMC_FreeMset(&serialized[pos]); } PyMem_Free(serialized); } return retval; } static PyObject *_PylibMC_RunCasCommand(PylibMC_Client *self, PyObject *args, PyObject *kwds) { /* function called by the set/add/etc commands */ static char *kws[] = { "key", "val", "cas", "time", NULL }; PyObject *ret = NULL; PyObject *key; PyObject *value; uint64_t cas = 0; unsigned int time = 0; /* this will be turned into a time_t */ bool success = false; memcached_return rc; if (!PyArg_ParseTupleAndKeywords(args, kwds, "SOL|I", kws, &key, &value, &cas, &time)) { return NULL; } if (!memcached_behavior_get(self->mc, MEMCACHED_BEHAVIOR_SUPPORT_CAS)) { PyErr_SetString(PyExc_ValueError, "cas without cas behavior"); return NULL; } pylibmc_mset mset = { NULL }; /* TODO: because it's RunSetCommand that does the zlib compression, cas can't currently use compressed values. */ success = _PylibMC_SerializeValue(key, NULL, value, time, &mset); if (!success || PyErr_Occurred() != NULL) { goto cleanup; } Py_BEGIN_ALLOW_THREADS; rc = memcached_cas(self->mc, mset.key, mset.key_len, mset.value, mset.value_len, mset.time, mset.flags, cas); Py_END_ALLOW_THREADS; switch(rc) { case MEMCACHED_SUCCESS: Py_INCREF(Py_True); ret = Py_True; break; case MEMCACHED_DATA_EXISTS: Py_INCREF(Py_False); ret = Py_False; break; default: PylibMC_ErrFromMemcachedWithKey(self, "memcached_cas", rc, mset.key, mset.key_len); } cleanup: _PylibMC_FreeMset(&mset); return ret; } static void _PylibMC_FreeMset(pylibmc_mset *mset) { Py_XDECREF(mset->key_obj); mset->key_obj = NULL; Py_XDECREF(mset->prefixed_key_obj); mset->prefixed_key_obj = NULL; /* Either a ref we own, or a ref passed to us which we borrowed. */ Py_XDECREF(mset->value_obj); mset->value_obj = NULL; } static int _PylibMC_SerializeValue(PyObject* key_obj, PyObject* key_prefix, PyObject* value_obj, time_t time, pylibmc_mset* serialized) { /* first zero the whole structure out */ memset((void *)serialized, 0x0, sizeof(pylibmc_mset)); serialized->time = time; serialized->success = false; serialized->flags = PYLIBMC_FLAG_NONE; if(!_PylibMC_CheckKey(key_obj) || PyString_AsStringAndSize(key_obj, &serialized->key, &serialized->key_len) == -1) { return false; } /* We need to incr our reference here so that it's guaranteed to exist while we release the GIL. Even if we fail after this it should be decremeneted by pylib_mset_free */ Py_INCREF(key_obj); serialized->key_obj = key_obj; /* Check the key_prefix */ if (key_prefix != NULL) { if (!_PylibMC_CheckKey(key_prefix)) { return false; } /* Ignore empty prefixes */ if (!PyString_Size(key_prefix)) { key_prefix = NULL; } } /* Make the prefixed key if appropriate */ if (key_prefix != NULL) { PyObject* prefixed_key_obj = NULL; /* freed by _PylibMC_FreeMset */ prefixed_key_obj = PyString_FromFormat("%s%s", PyString_AS_STRING(key_prefix), PyString_AS_STRING(key_obj)); if(prefixed_key_obj == NULL) { return false; } /* check the key and overwrite the C string */ if(!_PylibMC_CheckKey(prefixed_key_obj) || PyString_AsStringAndSize(prefixed_key_obj, &serialized->key, &serialized->key_len) == -1) { Py_DECREF(prefixed_key_obj); return false; } serialized->prefixed_key_obj = prefixed_key_obj; } /* Key/key_size should be harmonized, now onto the value */ PyObject* store_val = NULL; /* First build store_val, a Python String object, out of the object we were passed */ if (PyString_Check(value_obj)) { store_val = value_obj; Py_INCREF(store_val); /* because we'll be decring it again in pylibmc_mset_free*/ } else if (PyBool_Check(value_obj)) { serialized->flags |= PYLIBMC_FLAG_BOOL; PyObject* tmp = PyNumber_Int(value_obj); store_val = PyObject_Str(tmp); Py_DECREF(tmp); } else if (PyInt_Check(value_obj)) { serialized->flags |= PYLIBMC_FLAG_INTEGER; PyObject* tmp = PyNumber_Int(value_obj); store_val = PyObject_Str(tmp); Py_DECREF(tmp); } else if (PyLong_Check(value_obj)) { serialized->flags |= PYLIBMC_FLAG_LONG; PyObject* tmp = PyNumber_Long(value_obj); store_val = PyObject_Str(tmp); Py_DECREF(tmp); } else if(value_obj != NULL) { /* we have no idea what it is, so we'll store it pickled */ Py_INCREF(value_obj); serialized->flags |= PYLIBMC_FLAG_PICKLE; store_val = _PylibMC_Pickle(value_obj); Py_DECREF(value_obj); } if (store_val == NULL) { return false; } if (PyString_AsStringAndSize(store_val, &serialized->value, &serialized->value_len) == -1) { if (serialized->flags == PYLIBMC_FLAG_NONE) { /* For some reason we weren't able to extract the value/size from a string that we were explicitly passed, that we INCREF'd above */ Py_DECREF(store_val); } return false; } /* So now we have a reference to a string that we may have created. we need that to keep existing while we release the GIL, so we need to hold the reference, but we need to free it up when we're done */ serialized->value_obj = store_val; return true; } /* {{{ Set commands (set, replace, add, prepend, append) */ static bool _PylibMC_RunSetCommand(PylibMC_Client* self, _PylibMC_SetCommand f, char *fname, pylibmc_mset* msets, size_t nkeys, size_t min_compress) { memcached_st *mc = self->mc; memcached_return rc = MEMCACHED_SUCCESS; int pos; bool error = false; bool allsuccess = true; Py_BEGIN_ALLOW_THREADS; for (pos=0; pos < nkeys && !error; pos++) { pylibmc_mset *mset = &msets[pos]; char *value = mset->value; size_t value_len = (size_t)mset->value_len; uint32_t flags = mset->flags; #ifdef USE_ZLIB char *compressed_value = NULL; size_t compressed_len = 0; if (min_compress && value_len >= min_compress) { Py_BLOCK_THREADS; _PylibMC_Deflate(value, value_len, &compressed_value, &compressed_len); Py_UNBLOCK_THREADS; } if (compressed_value != NULL) { /* Will want to change this if this function * needs to get back at the old *value at some point */ value = compressed_value; value_len = compressed_len; flags |= PYLIBMC_FLAG_ZLIB; } #endif /* Finally go and call the actual libmemcached function */ if (mset->key_len == 0) { /* Most other implementations ignore zero-length keys, so we'll just do that */ rc = MEMCACHED_NOTSTORED; } else { rc = f(mc, mset->key, mset->key_len, value, value_len, mset->time, flags); } #ifdef USE_ZLIB if (compressed_value != NULL) { free(compressed_value); } #endif switch (rc) { case MEMCACHED_SUCCESS: mset->success = true; break; case MEMCACHED_FAILURE: case MEMCACHED_NO_KEY_PROVIDED: case MEMCACHED_BAD_KEY_PROVIDED: case MEMCACHED_MEMORY_ALLOCATION_FAILURE: case MEMCACHED_DATA_EXISTS: case MEMCACHED_NOTSTORED: mset->success = false; allsuccess = false; break; default: mset->success = false; allsuccess = false; error = true; /* will break the for loop */ } /* switch */ } /* end for each mset */ Py_END_ALLOW_THREADS; /* We only return the last return value, even for a _multi operation, but we do set the success on the mset */ if (error) { PylibMC_ErrFromMemcached(self, fname, rc); return false; } else { return allsuccess; } } /* These all just call _PylibMC_RunSetCommand with the appropriate * arguments. In other words: bulk. */ static PyObject *PylibMC_Client_set(PylibMC_Client *self, PyObject *args, PyObject *kwds) { PyObject* retval = _PylibMC_RunSetCommandSingle( self, memcached_set, "memcached_set", args, kwds); return retval; } static PyObject *PylibMC_Client_replace(PylibMC_Client *self, PyObject *args, PyObject *kwds) { return _PylibMC_RunSetCommandSingle( self, memcached_replace, "memcached_replace", args, kwds); } static PyObject *PylibMC_Client_add(PylibMC_Client *self, PyObject *args, PyObject *kwds) { return _PylibMC_RunSetCommandSingle( self, memcached_add, "memcached_add", args, kwds); } static PyObject *PylibMC_Client_prepend(PylibMC_Client *self, PyObject *args, PyObject *kwds) { return _PylibMC_RunSetCommandSingle( self, memcached_prepend, "memcached_prepend", args, kwds); } static PyObject *PylibMC_Client_append(PylibMC_Client *self, PyObject *args, PyObject *kwds) { return _PylibMC_RunSetCommandSingle( self, memcached_append, "memcached_append", args, kwds); } /* }}} */ static PyObject *PylibMC_Client_cas(PylibMC_Client *self, PyObject *args, PyObject *kwds) { return _PylibMC_RunCasCommand(self, args, kwds); } static PyObject *PylibMC_Client_delete(PylibMC_Client *self, PyObject *args) { char *key; Py_ssize_t key_len = 0; memcached_return rc; if (PyArg_ParseTuple(args, "s#:delete", &key, &key_len) && _PylibMC_CheckKeyStringAndSize(key, key_len)) { Py_BEGIN_ALLOW_THREADS; rc = memcached_delete(self->mc, key, key_len, 0); Py_END_ALLOW_THREADS; switch (rc) { case MEMCACHED_SUCCESS: Py_RETURN_TRUE; case MEMCACHED_FAILURE: case MEMCACHED_NOTFOUND: case MEMCACHED_NO_KEY_PROVIDED: case MEMCACHED_BAD_KEY_PROVIDED: Py_RETURN_FALSE; default: return PylibMC_ErrFromMemcachedWithKey(self, "memcached_delete", rc, key, key_len); } } return NULL; } /* {{{ Increment & decrement */ static PyObject *_PylibMC_IncrSingle(PylibMC_Client *self, _PylibMC_IncrCommand incr_func, PyObject *args) { char *key; Py_ssize_t key_len = 0; int delta = 1; if (!PyArg_ParseTuple(args, "s#|i", &key, &key_len, &delta)) { return NULL; } else if (!_PylibMC_CheckKeyStringAndSize(key, key_len)) { return NULL; } if (delta < 0L) { PyErr_SetString(PyExc_ValueError, "delta must be positive"); return NULL; } if ((unsigned int)delta != delta) { PyErr_Format(PyExc_OverflowError, "%d", delta); return NULL; } pylibmc_incr incr = { key, key_len, incr_func, delta, 0 }; _PylibMC_IncrDecr(self, &incr, 1); if(PyErr_Occurred() != NULL) { /* exception already on the stack */ return NULL; } /* might be NULL, but if that's true then it's the right return value */ return PyLong_FromUnsignedLong((unsigned long)incr.result); } static PyObject *_PylibMC_IncrMulti(PylibMC_Client *self, _PylibMC_IncrCommand incr_func, PyObject *args, PyObject *kwds) { PyObject *key = NULL; PyObject *keys = NULL; PyObject *keys_tmp = NULL; PyObject *key_prefix = NULL; PyObject *retval = NULL; PyObject *iterator = NULL; unsigned int delta = 1; size_t nkeys = 0, i = 0; static char *kws[] = { "keys", "key_prefix", "delta", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|SI", kws, &keys, &key_prefix, &delta)) return NULL; nkeys = (size_t)PySequence_Size(keys); if (nkeys == -1) return NULL; if (key_prefix != NULL) { if (!_PylibMC_CheckKey(key_prefix)) return NULL; if (PyString_Size(key_prefix) == 0) key_prefix = NULL; } keys_tmp = PyList_New(nkeys); if (keys_tmp == NULL) return NULL; pylibmc_incr *incrs = PyMem_New(pylibmc_incr, nkeys); if (incrs == NULL) goto cleanup; iterator = PyObject_GetIter(keys); if (iterator == NULL) goto cleanup; /* Build pylibmc_incr structs, prefixed as appropriate. */ for (i = 0; (key = PyIter_Next(iterator)) != NULL; i++) { pylibmc_incr *incr = incrs + i; if (!_PylibMC_CheckKey(key)) goto loopcleanup; /* prefix `key` with `key_prefix` */ if (key_prefix != NULL) { PyObject* newkey = PyString_FromFormat("%s%s", PyString_AS_STRING(key_prefix), PyString_AS_STRING(key)); Py_DECREF(key); key = newkey; } Py_INCREF(key); if (PyList_SetItem(keys_tmp, i, key) == -1) goto loopcleanup; /* Populate pylibmc_incr */ if (PyString_AsStringAndSize(key, &incr->key, &incr->key_len) == -1) goto loopcleanup; incr->delta = delta; incr->incr_func = incr_func; /* After incring we have no way of knowing whether the real result is 0 * or if the incr wasn't successful (or if noreply is set), but since * we're not actually returning the result that's okay for now */ incr->result = 0; loopcleanup: Py_DECREF(key); if (PyErr_Occurred()) goto cleanup; } /* end each key */ _PylibMC_IncrDecr(self, incrs, nkeys); if (!PyErr_Occurred()) { retval = Py_None; Py_INCREF(retval); } else { Py_XDECREF(retval); retval = NULL; } cleanup: if (incrs != NULL) PyMem_Free(incrs); Py_DECREF(keys_tmp); Py_XDECREF(iterator); return retval; } static PyObject *PylibMC_Client_incr(PylibMC_Client *self, PyObject *args) { return _PylibMC_IncrSingle(self, memcached_increment, args); } static PyObject *PylibMC_Client_decr(PylibMC_Client *self, PyObject *args) { return _PylibMC_IncrSingle(self, memcached_decrement, args); } static PyObject *PylibMC_Client_incr_multi(PylibMC_Client *self, PyObject *args, PyObject *kwds) { return _PylibMC_IncrMulti(self, memcached_increment, args, kwds); } static bool _PylibMC_IncrDecr(PylibMC_Client *self, pylibmc_incr *incrs, size_t nkeys) { memcached_return rc = MEMCACHED_SUCCESS; _PylibMC_IncrCommand f = NULL; size_t i, notfound = 0, errors = 0; Py_BEGIN_ALLOW_THREADS; for (i = 0; i < nkeys; i++) { pylibmc_incr *incr = &incrs[i]; uint64_t result = 0; f = incr->incr_func; rc = f(self->mc, incr->key, incr->key_len, incr->delta, &result); /* TODO Signal errors through `incr` */ if (rc == MEMCACHED_SUCCESS) { incr->result = result; } else if (rc == MEMCACHED_NOTFOUND) { notfound++; } else { errors++; } } Py_END_ALLOW_THREADS; if (errors + notfound) { PyObject *exc = PylibMCExc_MemcachedError; if (errors == 0) exc = _exc_by_rc(MEMCACHED_NOTFOUND); else if (errors == 1) exc = _exc_by_rc(rc); PyErr_Format(exc, "%d keys %s", (int)(notfound + errors), errors ? "failed" : "not found"); } return 0 == (errors + notfound); } /* }}} */ memcached_return pylibmc_memcached_fetch_multi( memcached_st *mc, char **keys, size_t nkeys, size_t *key_lens, memcached_result_st **results, size_t *nresults, char **err_func) { /** * Completely GIL-free multi getter * * Takes a set of keys given by *keys*, and stuffs the results into heap * memory returned by *results*. * * If an error occured during retrieval, this function returns * non-MEMCACHED_SUCCESS and *err_func* will point to a useful error * function name. * * FIXME *results* is expected to be able to hold one more result than * there are keys asked for, because of an implementation detail. */ memcached_return rc; *err_func = NULL; rc = memcached_mget(mc, (const char **)keys, key_lens, nkeys); if (rc != MEMCACHED_SUCCESS) { *err_func = "memcached_mget"; return rc; } /* Allocate as much as could possibly be needed, and an extra because of * how libmemcached signals EOF. */ *results = PyMem_New(memcached_result_st, nkeys + 1); /* Note that nresults will not be off by one with this because the loops * runs a half pass after the last key has been fetched, thus bumping the * count once. */ for (*nresults = 0; ; (*nresults)++) { memcached_result_st *res = memcached_result_create(mc, *results + *nresults); /* if loop spins out of control, this fails */ assert(nkeys >= (*nresults)); res = memcached_fetch_result(mc, res, &rc); if (res == NULL || rc == MEMCACHED_END) { /* This is how libmecached signals EOF. */ break; } else if (rc == MEMCACHED_BAD_KEY_PROVIDED || rc == MEMCACHED_NO_KEY_PROVIDED) { continue; } else if (rc != MEMCACHED_SUCCESS) { memcached_quit(mc); /* Reset fetch state */ *err_func = "memcached_fetch"; /* Clean-up procedure */ do { memcached_result_free(*results + *nresults); } while ((*nresults)--); PyMem_Free(*results); *results = NULL; *nresults = 0; return rc; } } return MEMCACHED_SUCCESS; } static PyObject *PylibMC_Client_get_multi( PylibMC_Client *self, PyObject *args, PyObject *kwds) { PyObject *key_seq, **key_objs, *retval = NULL; char **keys, *prefix = NULL; char *err_func = NULL; memcached_result_st *res, *results = NULL; Py_ssize_t prefix_len = 0; Py_ssize_t i; PyObject *key_it, *ckey; size_t *key_lens; size_t nkeys, nresults = 0; memcached_return rc; static char *kws[] = { "keys", "key_prefix", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s#:get_multi", kws, &key_seq, &prefix, &prefix_len)) return NULL; if ((nkeys = (size_t)PySequence_Length(key_seq)) == -1) return NULL; /* Populate keys and key_lens. */ keys = PyMem_New(char *, nkeys); key_lens = PyMem_New(size_t, nkeys); key_objs = PyMem_New(PyObject *, nkeys); if (!keys || !key_lens || !key_objs) { PyMem_Free(keys); PyMem_Free(key_lens); PyMem_Free(key_objs); return PyErr_NoMemory(); } /* Clear potential previous exception, because we explicitly check for * exceptions as a loop predicate. */ PyErr_Clear(); /* Iterate through all keys and set lengths etc. */ key_it = PyObject_GetIter(key_seq); i = 0; while ((ckey = PyIter_Next(key_it)) != NULL) { char *key; Py_ssize_t key_len; PyObject *rkey; assert(i < nkeys); if (PyErr_Occurred() || !_PylibMC_CheckKey(ckey)) { nkeys = i; goto earlybird; } PyString_AsStringAndSize(ckey, &key, &key_len); key_lens[i] = (size_t)(key_len + prefix_len); /* Skip empty keys */ if (!key_lens[i]) { Py_DECREF(ckey); continue; } /* determine rkey, the prefixed ckey */ if (prefix != NULL) { rkey = PyString_FromStringAndSize(prefix, prefix_len); PyString_Concat(&rkey, ckey); if (rkey == NULL) goto earlybird; Py_DECREF(rkey); rkey = PyString_FromFormat("%s%s", prefix, PyString_AS_STRING(ckey)); } else { Py_INCREF(ckey); rkey = ckey; } Py_DECREF(ckey); keys[i] = PyString_AS_STRING(rkey); key_objs[i++] = rkey; } nkeys = i; Py_XDECREF(key_it); if (nkeys == 0) { retval = PyDict_New(); goto earlybird; } else if (PyErr_Occurred()) { nkeys--; goto earlybird; } /* TODO Make an iterator interface for getting each key separately. * * This would help large retrievals, as a single dictionary containing all * the data at once isn't needed. (Should probably look into if it's even * worth it.) */ Py_BEGIN_ALLOW_THREADS; rc = pylibmc_memcached_fetch_multi(self->mc, keys, nkeys, key_lens, &results, &nresults, &err_func); Py_END_ALLOW_THREADS; if (rc != MEMCACHED_SUCCESS) { PylibMC_ErrFromMemcached(self, err_func, rc); goto earlybird; } retval = PyDict_New(); for (i = 0; i < nresults; i++) { PyObject *val, *key_obj; int rc; res = results + i; /* Long-winded, but this way we can handle NUL-bytes in keys. */ key_obj = PyString_FromStringAndSize(memcached_result_key_value(res) + prefix_len, memcached_result_key_length(res) - prefix_len); if (key_obj == NULL) goto unpack_error; /* Parse out value */ val = _PylibMC_parse_memcached_result(res); if (val == NULL) goto unpack_error; rc = PyDict_SetItem(retval, key_obj, val); Py_DECREF(key_obj); Py_DECREF(val); if (rc != 0) goto unpack_error; continue; unpack_error: Py_DECREF(retval); retval = NULL; break; } earlybird: PyMem_Free(key_lens); PyMem_Free(keys); for (i = 0; i < nkeys; i++) Py_DECREF(key_objs[i]); PyMem_Free(key_objs); if (results != NULL) { for (i = 0; i < nresults && results != NULL; i++) { memcached_result_free(results + i); } PyMem_Free(results); } /* Not INCREFing because the only two outcomes are NULL and a new dict. * We're the owner of that dict already, so. */ return retval; } /** * Run func over every item in value, building arguments of: * *(item + extra_args) */ static PyObject *_PylibMC_DoMulti(PyObject *values, PyObject *func, PyObject *prefix, PyObject *extra_args) { /* TODO: acquire/release the GIL only once per DoMulti rather than once per action; fortunately this is only used for delete_multi, which isn't used very often */ PyObject *retval = PyList_New(0); PyObject *iter = NULL; PyObject *item = NULL; int is_mapping = PyMapping_Check(values); if (retval == NULL) goto error; if ((iter = PyObject_GetIter(values)) == NULL) goto error; while ((item = PyIter_Next(iter)) != NULL) { PyObject *args_f = NULL; PyObject *args = NULL; PyObject *key = NULL; PyObject *ro = NULL; /* Calculate key. */ if (prefix == NULL || prefix == Py_None) { /* We now have two owned references to item. */ key = item; Py_INCREF(key); } else { key = PySequence_Concat(prefix, item); } if (key == NULL || !_PylibMC_CheckKey(key)) goto iter_error; /* Calculate args. */ if (is_mapping) { PyObject *value; char *key_str = PyString_AS_STRING(item); if ((value = PyMapping_GetItemString(values, key_str)) == NULL) goto iter_error; args = PyTuple_Pack(2, key, value); Py_DECREF(value); } else { args = PyTuple_Pack(1, key); } if (args == NULL) goto iter_error; /* Calculate full argument tuple. */ if (extra_args == NULL) { Py_INCREF(args); args_f = args; } else { if ((args_f = PySequence_Concat(args, extra_args)) == NULL) goto iter_error; } /* Call stuff. */ ro = PyObject_CallObject(func, args_f); /* This is actually safe even if True got deleted because we're * only comparing addresses. */ Py_XDECREF(ro); if (ro == NULL) { goto iter_error; } else if (ro != Py_True) { if (PyList_Append(retval, item) != 0) goto iter_error; } Py_DECREF(args_f); Py_DECREF(args); Py_DECREF(key); Py_DECREF(item); continue; iter_error: Py_XDECREF(args_f); Py_XDECREF(args); Py_XDECREF(key); Py_DECREF(item); goto error; } Py_DECREF(iter); return retval; error: Py_XDECREF(retval); Py_XDECREF(iter); return NULL; } static PyObject *PylibMC_Client_set_multi(PylibMC_Client *self, PyObject *args, PyObject *kwds) { return _PylibMC_RunSetCommandMulti(self, memcached_set, "memcached_set_multi", args, kwds); } static PyObject *PylibMC_Client_add_multi(PylibMC_Client *self, PyObject *args, PyObject *kwds) { return _PylibMC_RunSetCommandMulti(self, memcached_add, "memcached_add_multi", args, kwds); } static PyObject *PylibMC_Client_delete_multi(PylibMC_Client *self, PyObject *args, PyObject *kwds) { PyObject *prefix = NULL; PyObject *time = NULL; PyObject *delete; PyObject *keys; PyObject *call_args; PyObject *retval; static char *kws[] = { "keys", "key_prefix", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|S:delete_multi", kws, &keys, &prefix)) return NULL; /** * Because of how DoMulti works, we have to prohibit the use of mappings * here. Otherwise, the values of the mapping will be the second argument * to the delete function; the time argument will then become a third * argument, which delete doesn't take. * * So a mapping to DoMulti would produce calls like: * DoMulti({"a": 1, "b": 2}, time=3) * delete("a", 1, 3) * delete("b", 2, 3) */ if (PyMapping_Check(keys)) { PyErr_SetString(PyExc_TypeError, "keys must be a sequence, not a mapping"); return NULL; } if ((delete = PyObject_GetAttrString((PyObject *)self, "delete")) == NULL) return NULL; if (time == NULL) { retval = _PylibMC_DoMulti(keys, delete, prefix, NULL); } else { if ((call_args = PyTuple_Pack(1, time)) == NULL) goto error; retval = _PylibMC_DoMulti(keys, delete, prefix, call_args); Py_DECREF(call_args); } Py_DECREF(delete); if (retval == NULL) return NULL; if (PyList_Size(retval) == 0) { Py_DECREF(retval); retval = Py_True; } else { Py_DECREF(retval); retval = Py_False; } Py_INCREF(retval); return retval; error: Py_XDECREF(delete); return NULL; } static PyObject *PylibMC_Client_get_behaviors(PylibMC_Client *self) { PyObject *retval = PyDict_New(); PylibMC_Behavior *b; if (retval == NULL) return NULL; for (b = PylibMC_behaviors; b->name != NULL; b++) { uint64_t bval; PyObject *x; bval = memcached_behavior_get(self->mc, b->flag); x = PyInt_FromLong((long)bval); if (x == NULL || PyDict_SetItemString(retval, b->name, x) == -1) { Py_XDECREF(x); goto error; } Py_DECREF(x); } return retval; error: Py_XDECREF(retval); return NULL; } static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *self, PyObject *behaviors) { PylibMC_Behavior *b; for (b = PylibMC_behaviors; b->name != NULL; b++) { PyObject *py_v; uint64_t v; memcached_return r; if (!PyMapping_HasKeyString(behaviors, b->name)) { continue; } else if ((py_v = PyMapping_GetItemString(behaviors, b->name)) == NULL) { goto error; } else if (!PyInt_Check(py_v)) { PyErr_Format(PyExc_TypeError, "behavior %.32s must be int", b->name); goto error; } v = (uint64_t)PyInt_AS_LONG(py_v); Py_DECREF(py_v); r = memcached_behavior_set(self->mc, b->flag, v); if (r != MEMCACHED_SUCCESS) { PyErr_Format(PylibMCExc_MemcachedError, "memcached_behavior_set returned %d for " "behavior '%.32s' = %u", r, b->name, (unsigned int)v); goto error; } } Py_RETURN_NONE; error: return NULL; } static memcached_return _PylibMC_AddServerCallback(memcached_st *mc, memcached_server_st *server, void *user) { _PylibMC_StatsContext *context = (_PylibMC_StatsContext *)user; PylibMC_Client *self = (PylibMC_Client *)context->self; memcached_stat_st *stat; memcached_return rc; PyObject *desc, *val; char **stat_keys = NULL; char **curr_key; stat = context->stats + context->index; if ((val = PyDict_New()) == NULL) return MEMCACHED_FAILURE; stat_keys = memcached_stat_get_keys(mc, stat, &rc); if (rc != MEMCACHED_SUCCESS) return rc; for (curr_key = stat_keys; *curr_key; curr_key++) { PyObject *curr_value; char *mc_val; int fail; mc_val = memcached_stat_get_value(mc, stat, *curr_key, &rc); if (rc != MEMCACHED_SUCCESS) { PylibMC_ErrFromMemcached(self, "get_stats val", rc); goto error; } curr_value = PyString_FromString(mc_val); free(mc_val); if (curr_value == NULL) goto error; fail = PyDict_SetItemString(val, *curr_key, curr_value); Py_DECREF(curr_value); if (fail) goto error; } free(stat_keys); desc = PyString_FromFormat("%s:%d (%u)", server->hostname, server->port, (unsigned int)context->index); PyList_SET_ITEM(context->retval, context->index++, Py_BuildValue("NN", desc, val)); return MEMCACHED_SUCCESS; error: free(stat_keys); Py_DECREF(val); return MEMCACHED_FAILURE; } static PyObject *PylibMC_Client_get_stats(PylibMC_Client *self, PyObject *args) { memcached_stat_st *stats; memcached_return rc; char *mc_args; Py_ssize_t nservers; _PylibMC_StatsContext context; mc_args = NULL; if (!PyArg_ParseTuple(args, "|s:get_stats", &mc_args)) return NULL; Py_BEGIN_ALLOW_THREADS; stats = memcached_stat(self->mc, mc_args, &rc); Py_END_ALLOW_THREADS; if (rc != MEMCACHED_SUCCESS) return PylibMC_ErrFromMemcached(self, "get_stats", rc); /** retval contents: * [(' ()', {stat: stat, stat: stat}), * (str, dict), * (str, dict)] */ nservers = (Py_ssize_t)memcached_server_count(self->mc); /* Setup stats callback context */ context.self = (PyObject *)self; context.retval = PyList_New(nservers); context.stats = stats; context.servers = NULL; /* DEPRECATED */ context.index = 0; #if LIBMEMCACHED_VERSION_HEX >= 0x00038000 memcached_server_function callbacks[] = { (memcached_server_function)_PylibMC_AddServerCallback }; rc = memcached_server_cursor(self->mc, callbacks, (void *)&context, 1); #else /* ver < libmemcached 0.38 */ context.servers = memcached_server_list(self->mc); for (; context.index < nservers; context.index++) { memcached_server_st *server = context.servers + context.index; memcached_return rc; rc = _PylibMC_AddServerCallback(self->mc, server, (void *)&context); if (rc != MEMCACHED_SUCCESS) break; } #endif if (rc != MEMCACHED_SUCCESS) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_RuntimeError, "unknown error occured"); Py_DECREF(context.retval); context.retval = NULL; } free(context.stats); return context.retval; } static PyObject *PylibMC_Client_flush_all(PylibMC_Client *self, PyObject *args, PyObject *kwds) { memcached_return rc; time_t expire = 0; PyObject *time = NULL; static char *kws[] = { "time", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O!:flush_all", kws, &PyInt_Type, &time)) return NULL; if (time != NULL) expire = PyInt_AS_LONG(time); expire = (expire > 0) ? expire : 0; Py_BEGIN_ALLOW_THREADS; rc = memcached_flush(self->mc, expire); Py_END_ALLOW_THREADS; if (rc != MEMCACHED_SUCCESS) return PylibMC_ErrFromMemcached(self, "flush_all", rc); Py_RETURN_TRUE; } static PyObject *PylibMC_Client_disconnect_all(PylibMC_Client *self) { Py_BEGIN_ALLOW_THREADS; memcached_quit(self->mc); Py_END_ALLOW_THREADS; Py_RETURN_NONE; } static PyObject *PylibMC_Client_clone(PylibMC_Client *self) { /* Essentially this is a reimplementation of the allocator, only it uses a * cloned memcached_st for mc. */ PylibMC_Client *clone; clone = (PylibMC_Client *)PyType_GenericNew(self->ob_type, NULL, NULL); if (clone == NULL) { return NULL; } Py_BEGIN_ALLOW_THREADS; clone->mc = memcached_clone(NULL, self->mc); Py_END_ALLOW_THREADS; return (PyObject *)clone; } /* }}} */ static PyObject *_exc_by_rc(memcached_return rc) { PylibMC_McErr *err; for (err = PylibMCExc_mc_errs; err->name != NULL; err++) if (err->rc == rc) return err->exc; return (PyObject *)PylibMCExc_MemcachedError; } static char *_get_lead(memcached_st *mc, char *buf, int n, const char *what, memcached_return error, const char *key, Py_ssize_t len) { int sz = snprintf(buf, n, "error %d from %.32s", error, what); if (key != NULL && len) { sz += snprintf(buf+sz, n-sz, "(%.32s)", key); } return buf; } static void _set_error(memcached_st *mc, memcached_return error, char *lead) { if (error == MEMCACHED_ERRNO) { PyErr_Format(PylibMCExc_MemcachedError, "%s: %s", lead, strerror(errno)); } else if (error == MEMCACHED_SUCCESS) { PyErr_Format(PyExc_RuntimeError, "error == MEMCACHED_SUCCESS"); } else { PyObject *exc = _exc_by_rc(error); #if LIBMEMCACHED_VERSION_HEX >= 0x00049000 PyErr_Format(exc, "%s: %.200s", lead, memcached_last_error_message(mc)); #else PyErr_Format(exc, "%s: %.200s", lead, memcached_strerror(mc, error)); #endif } } static PyObject *PylibMC_ErrFromMemcachedWithKey(PylibMC_Client *self, const char *what, memcached_return error, const char *key, Py_ssize_t len) { char lead[128]; _get_lead(self->mc, lead, sizeof(lead), what, error, key, len); _set_error(self->mc, error, lead); return NULL; } static PyObject *PylibMC_ErrFromMemcached(PylibMC_Client *self, const char *what, memcached_return error) { char lead[128]; _get_lead(self->mc, lead, sizeof(lead), what, error, NULL, 0); _set_error(self->mc, error, lead); return NULL; } /* {{{ Pickling */ static PyObject *_PylibMC_GetPickles(const char *attname) { PyObject *pickle, *pickle_attr; pickle_attr = NULL; /* Import cPickle or pickle. */ pickle = PyImport_ImportModule("cPickle"); if (pickle == NULL) { PyErr_Clear(); pickle = PyImport_ImportModule("pickle"); } /* Find attribute and return it. */ if (pickle != NULL) { pickle_attr = PyObject_GetAttrString(pickle, attname); Py_DECREF(pickle); } return pickle_attr; } static PyObject *_PylibMC_Unpickle(const char *buff, size_t size) { PyObject *pickle_load; PyObject *retval = NULL; retval = NULL; pickle_load = _PylibMC_GetPickles("loads"); if (pickle_load != NULL) { retval = PyObject_CallFunction(pickle_load, "s#", buff, size); Py_DECREF(pickle_load); } return retval; } static PyObject *_PylibMC_Pickle(PyObject *val) { PyObject *pickle_dump; PyObject *retval = NULL; pickle_dump = _PylibMC_GetPickles("dumps"); if (pickle_dump != NULL) { retval = PyObject_CallFunction(pickle_dump, "Oi", val, -1); Py_DECREF(pickle_dump); } return retval; } /* }}} */ static int _PylibMC_CheckKey(PyObject *key) { if (key == NULL) { PyErr_SetString(PyExc_ValueError, "key must be given"); return 0; } else if (!PyString_Check(key)) { PyErr_SetString(PyExc_TypeError, "key must be an instance of str"); return 0; } return _PylibMC_CheckKeyStringAndSize( PyString_AS_STRING(key), PyString_GET_SIZE(key)); } static int _PylibMC_CheckKeyStringAndSize(char *key, Py_ssize_t size) { /* libmemcached pads max_key_size with one byte for null termination */ if (size >= MEMCACHED_MAX_KEY) { PyErr_Format(PyExc_ValueError, "key too long, max is %d", MEMCACHED_MAX_KEY - 1); return 0; #ifdef NO_EMPTY_KEYS /* I wish... */ } else if (size == 0) { PyErr_Format(PyExc_ValueError, "key cannot be empty"); #endif } /* TODO Check key contents here. */ return key != NULL; } static int _init_sasl(void) { #if LIBMEMCACHED_WITH_SASL_SUPPORT int rc; /* sasl_client_init needs to be called once before using SASL, and * sasl_done after all SASL usage is done (so basically, once per process * lifetime). */ rc = sasl_client_init(NULL); if (rc == SASL_NOMEM) { PyErr_NoMemory(); } else if (rc == SASL_BADVERS) { PyErr_Format(PyExc_RuntimeError, "SASL: Mechanism version mismatch"); } else if (rc == SASL_BADPARAM) { PyErr_Format(PyExc_RuntimeError, "SASL: Error in config file"); } else if (rc == SASL_NOMECH) { PyErr_Format(PyExc_RuntimeError, "SASL: No mechanisms available"); } else if (rc != SASL_OK) { PyErr_Format(PyExc_RuntimeError, "SASL: Unknown error (rc=%d)", rc); } if (rc != SASL_OK) { return false; } /* Terrible, terrible hack. Need to call sasl_done, but the Python/C API * doesn't provide a hook for when the module is unloaded, so register an * atexit handler. This is particularly problematic because * "At most 32 cleanup functions can be registered". */ if (Py_AtExit(sasl_done)) { PyErr_Format(PyExc_RuntimeError, "Failed to register atexit handler"); return false; } #endif return true; } static int _check_libmemcached_version(void) { uint8_t maj, min; char *ver, *dot, *tmp; ver = dot = strdup(LIBMEMCACHED_VERSION_STRING); while ((tmp = strrchr(ver, '.')) != NULL) { dot = tmp; *dot = 0; } maj = atoi(ver); min = atoi(dot + 1); if (maj == 0 && min < 32) { PyErr_Format(PyExc_RuntimeError, "pylibmc requires >= libmemcached 0.32, was compiled with %s", LIBMEMCACHED_VERSION_STRING); return false; } else { return true; } } static void _make_excs(PyObject *module) { PyObject *exc_objs; PylibMC_McErr *err; PylibMCExc_MemcachedError = PyErr_NewException( "_pylibmc.MemcachedError", NULL, NULL); exc_objs = PyList_New(0); PyList_Append(exc_objs, Py_BuildValue("sO", "Error", (PyObject *)PylibMCExc_MemcachedError)); for (err = PylibMCExc_mc_errs; err->name != NULL; err++) { char excnam[64]; snprintf(excnam, 64, "_pylibmc.%s", err->name); err->exc = PyErr_NewException(excnam, PylibMCExc_MemcachedError, NULL); PyObject_SetAttrString(err->exc, "retcode", PyInt_FromLong(err->rc)); PyModule_AddObject(module, err->name, (PyObject *)err->exc); PyList_Append(exc_objs, Py_BuildValue("sO", err->name, (PyObject *)err->exc)); } PyModule_AddObject(module, "MemcachedError", (PyObject *)PylibMCExc_MemcachedError); PyModule_AddObject(module, "exceptions", exc_objs); } static void _make_behavior_consts(PyObject *mod) { PyObject *behavior_names; PylibMC_Behavior *b; char name[128]; /* Add hasher and distribution constants. */ for (b = PylibMC_hashers; b->name != NULL; b++) { sprintf(name, "hash_%s", b->name); PyModule_AddIntConstant(mod, name, b->flag); } for (b = PylibMC_distributions; b->name != NULL; b++) { sprintf(name, "distribution_%s", b->name); PyModule_AddIntConstant(mod, name, b->flag); } behavior_names = PyList_New(0); for (b = PylibMC_behaviors; b->name != NULL; b++) { PyList_Append(behavior_names, PyString_FromString(b->name)); } PyModule_AddObject(mod, "all_behaviors", behavior_names); } static PyMethodDef PylibMC_functions[] = { {NULL, NULL, 0, NULL} }; PyMODINIT_FUNC init_pylibmc(void) { PyObject *module; if (!_check_libmemcached_version()) return; if (!_init_sasl()) return; if (PyType_Ready(&PylibMC_ClientType) < 0) { return; } module = Py_InitModule3("_pylibmc", PylibMC_functions, "Hand-made wrapper for libmemcached.\n\ \n\ You should really use the Python wrapper around this library.\n\ \n\ c = _pylibmc.client([(_pylibmc.server_type_tcp, 'localhost', 11211)])\n\ \n\ Three-tuples of (type, host, port) are used. If type is `server_type_unix`,\n\ no port should be given. libmemcached can parse strings as well::\n\ \n\ c = _pylibmc.client('localhost')\n\ \n\ See libmemcached's memcached_servers_parse for more info on that. I'm told \n\ you can use UNIX domain sockets by specifying paths, and multiple servers \n\ by using comma-separation. Good luck with that.\n"); if (module == NULL) { return; } _make_excs(module); PyModule_AddStringConstant(module, "__version__", PYLIBMC_VERSION); PyModule_ADD_REF(module, "client", (PyObject *)&PylibMC_ClientType); PyModule_AddStringConstant(module, "libmemcached_version", LIBMEMCACHED_VERSION_STRING); #if LIBMEMCACHED_WITH_SASL_SUPPORT PyModule_ADD_REF(module, "support_sasl", Py_True); #else PyModule_ADD_REF(module, "support_sasl", Py_False); #endif #ifdef USE_ZLIB PyModule_ADD_REF(module, "support_compression", Py_True); #else PyModule_ADD_REF(module, "support_compression", Py_False); #endif PyModule_AddIntConstant(module, "server_type_tcp", PYLIBMC_SERVER_TCP); PyModule_AddIntConstant(module, "server_type_udp", PYLIBMC_SERVER_UDP); PyModule_AddIntConstant(module, "server_type_unix", PYLIBMC_SERVER_UNIX); _make_behavior_consts(module); } pylibmc-1.2.3/_pylibmcmodule.h0000644000076500000240000003674711746271101017313 0ustar lericsonstaff00000000000000/** * _pylibmc: hand-made libmemcached bindings for Python * * Copyright (c) 2008, Ludvig Ericson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * - Neither the name of the author nor the names of the contributors may be * used to endorse or promote products derived from this software without * specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef __PYLIBMC_H__ #define __PYLIBMC_H__ /* This makes the "s#" format for PyArg_ParseTuple and such take a Py_ssize_t * instead of an int or whatever. */ #define PY_SSIZE_T_CLEAN #include #include #ifndef LIBMEMCACHED_VERSION_HEX # define LIBMEMCACHED_VERSION_HEX 0x0 #endif #include "pylibmc-version.h" /* Py_ssize_t appeared in Python 2.5. */ #ifndef PY_SSIZE_T_MAX typedef ssize_t Py_ssize_t; #endif /* Server types. */ #define PYLIBMC_SERVER_TCP (1 << 0) #define PYLIBMC_SERVER_UDP (1 << 1) #define PYLIBMC_SERVER_UNIX (1 << 2) /* {{{ Key flags from python-memcached * Some flags (like the compression one, ZLIB) are combined with others. */ #define PYLIBMC_FLAG_NONE 0 #define PYLIBMC_FLAG_PICKLE (1 << 0) #define PYLIBMC_FLAG_INTEGER (1 << 1) #define PYLIBMC_FLAG_LONG (1 << 2) /* Note: this is an addition! python-memcached doesn't handle bools. */ #define PYLIBMC_FLAG_BOOL (1 << 4) #define PYLIBMC_FLAG_TYPES (PYLIBMC_FLAG_PICKLE | PYLIBMC_FLAG_INTEGER | \ PYLIBMC_FLAG_LONG | PYLIBMC_FLAG_BOOL) /* Modifier flags */ #define PYLIBMC_FLAG_ZLIB (1 << 3) /* }}} */ typedef memcached_return (*_PylibMC_SetCommand)(memcached_st *, const char *, size_t, const char *, size_t, time_t, uint32_t); typedef memcached_return (*_PylibMC_IncrCommand)(memcached_st *, const char *, size_t, unsigned int, uint64_t*); static PyObject *_exc_by_rc(memcached_return); typedef struct { char *key; Py_ssize_t key_len; char *value; Py_ssize_t value_len; time_t time; uint32_t flags; /* the objects that must be freed after the mset is executed */ PyObject *key_obj; PyObject *prefixed_key_obj; PyObject *value_obj; /* the success of executing the mset afterwards */ int success; } pylibmc_mset; typedef struct { char* key; Py_ssize_t key_len; _PylibMC_IncrCommand incr_func; unsigned int delta; uint64_t result; } pylibmc_incr; typedef struct { PyObject *self; PyObject *retval; memcached_server_st *servers; /* DEPRECATED */ memcached_stat_st *stats; int index; } _PylibMC_StatsContext; /* {{{ Exceptions */ static PyObject *PylibMCExc_MemcachedError; /* Mapping of memcached_return value -> Python exception object. */ typedef struct { memcached_return rc; char *name; PyObject *exc; } PylibMC_McErr; static PylibMC_McErr PylibMCExc_mc_errs[] = { { MEMCACHED_FAILURE, "Failure", NULL }, { MEMCACHED_HOST_LOOKUP_FAILURE, "HostLookupError", NULL }, { MEMCACHED_CONNECTION_FAILURE, "ConnectionError", NULL }, { MEMCACHED_CONNECTION_BIND_FAILURE, "ConnectionBindError", NULL }, { MEMCACHED_WRITE_FAILURE, "WriteError", NULL }, { MEMCACHED_READ_FAILURE, "ReadError", NULL }, { MEMCACHED_UNKNOWN_READ_FAILURE, "UnknownReadFailure", NULL }, { MEMCACHED_PROTOCOL_ERROR, "ProtocolError", NULL }, { MEMCACHED_CLIENT_ERROR, "ClientError", NULL }, { MEMCACHED_SERVER_ERROR, "ServerError", NULL }, { MEMCACHED_CONNECTION_SOCKET_CREATE_FAILURE, "SocketCreateError", NULL }, { MEMCACHED_DATA_EXISTS, "DataExists", NULL }, { MEMCACHED_DATA_DOES_NOT_EXIST, "DataDoesNotExist", NULL }, //{ MEMCACHED_NOTSTORED, "NotStored", NULL }, //{ MEMCACHED_STORED, "Stored", NULL }, { MEMCACHED_NOTFOUND, "NotFound", NULL }, { MEMCACHED_MEMORY_ALLOCATION_FAILURE, "AllocationError", NULL }, //{ MEMCACHED_PARTIAL_READ, "PartialRead", NULL }, { MEMCACHED_SOME_ERRORS, "SomeErrors", NULL }, { MEMCACHED_NO_SERVERS, "NoServers", NULL }, //{ MEMCACHED_END, "", NULL }, //{ MEMCACHED_DELETED, "", NULL }, //{ MEMCACHED_VALUE, "", NULL }, //{ MEMCACHED_STAT, "", NULL }, //{ MEMCACHED_ITEM, "", NULL }, //{ MEMCACHED_ERRNO, "", NULL }, { MEMCACHED_FAIL_UNIX_SOCKET, "UnixSocketError", NULL }, { MEMCACHED_NOT_SUPPORTED, "NotSupportedError", NULL }, { MEMCACHED_FETCH_NOTFINISHED, "FetchNotFinished", NULL }, //{ MEMCACHED_TIMEOUT, "TimeoutError", NULL }, //{ MEMCACHED_BUFFERED, "Buffer, NULL }, { MEMCACHED_BAD_KEY_PROVIDED, "BadKeyProvided", NULL }, { MEMCACHED_INVALID_HOST_PROTOCOL, "InvalidHostProtocolError", NULL }, { MEMCACHED_SERVER_MARKED_DEAD, "ServerDead", NULL }, #ifdef MEMCACHED_SERVER_TEMPORARILY_DISABLED { MEMCACHED_SERVER_TEMPORARILY_DISABLED, "ServerDown", NULL }, #endif { MEMCACHED_UNKNOWN_STAT_KEY, "UnknownStatKey", NULL }, //{ MEMCACHED_E2BIG, "TooBigError", NULL }, { 0, NULL, NULL } }; /* }}} */ /* {{{ Behavior statics */ typedef struct { int flag; char *name; } PylibMC_Behavior; static PylibMC_Behavior PylibMC_behaviors[] = { { MEMCACHED_BEHAVIOR_NO_BLOCK, "no_block" }, { MEMCACHED_BEHAVIOR_TCP_NODELAY, "tcp_nodelay" }, { MEMCACHED_BEHAVIOR_HASH, "hash" }, { MEMCACHED_BEHAVIOR_KETAMA_HASH, "ketama_hash" }, { MEMCACHED_BEHAVIOR_KETAMA, "ketama" }, { MEMCACHED_BEHAVIOR_KETAMA_WEIGHTED, "ketama_weighted" }, { MEMCACHED_BEHAVIOR_DISTRIBUTION, "distribution" }, { MEMCACHED_BEHAVIOR_SUPPORT_CAS, "cas" }, { MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, "buffer_requests" }, { MEMCACHED_BEHAVIOR_VERIFY_KEY, "verify_keys" }, { MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT, "connect_timeout" }, { MEMCACHED_BEHAVIOR_SND_TIMEOUT, "send_timeout" }, { MEMCACHED_BEHAVIOR_RCV_TIMEOUT, "receive_timeout" }, { MEMCACHED_BEHAVIOR_NUMBER_OF_REPLICAS, "num_replicas" }, { MEMCACHED_BEHAVIOR_AUTO_EJECT_HOSTS, "auto_eject" }, { MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, "retry_timeout" }, #if LIBMEMCACHED_VERSION_HEX >= 0x00049000 { MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, "remove_failed" }, #endif /* make sure failure_limit is set after remove_failed * as the latter overwrites the former. */ { MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT, "failure_limit" }, { MEMCACHED_BEHAVIOR_IO_MSG_WATERMARK, "_io_msg_watermark" }, { MEMCACHED_BEHAVIOR_IO_BYTES_WATERMARK, "_io_bytes_watermark" }, { MEMCACHED_BEHAVIOR_IO_KEY_PREFETCH, "_io_key_prefetch" }, { MEMCACHED_BEHAVIOR_HASH_WITH_PREFIX_KEY, "_hash_with_prefix_key" }, { MEMCACHED_BEHAVIOR_NOREPLY, "_noreply" }, { MEMCACHED_BEHAVIOR_SORT_HOSTS, "_sort_hosts" }, { MEMCACHED_BEHAVIOR_POLL_TIMEOUT, "_poll_timeout" }, { MEMCACHED_BEHAVIOR_SOCKET_SEND_SIZE, "_socket_send_size" }, { MEMCACHED_BEHAVIOR_SOCKET_RECV_SIZE, "_socket_recv_size" }, { 0, NULL } }; static PylibMC_Behavior PylibMC_hashers[] = { { MEMCACHED_HASH_DEFAULT, "default" }, { MEMCACHED_HASH_MD5, "md5" }, { MEMCACHED_HASH_CRC, "crc" }, { MEMCACHED_HASH_FNV1_64, "fnv1_64" }, { MEMCACHED_HASH_FNV1A_64, "fnv1a_64" }, { MEMCACHED_HASH_FNV1_32, "fnv1_32" }, { MEMCACHED_HASH_FNV1A_32, "fnv1a_32" }, { MEMCACHED_HASH_MURMUR, "murmur" }, #ifdef MEMCACHED_HASH_HSIEH { MEMCACHED_HASH_HSIEH, "hsieh" }, #endif { 0, NULL } }; static PylibMC_Behavior PylibMC_distributions[] = { { MEMCACHED_DISTRIBUTION_MODULA, "modula" }, { MEMCACHED_DISTRIBUTION_CONSISTENT, "consistent" }, { MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA, "consistent_ketama" }, { 0, NULL } }; /* }}} */ /* {{{ _pylibmc.client */ typedef struct { PyObject_HEAD memcached_st *mc; uint8_t sasl_set; } PylibMC_Client; /* {{{ Prototypes */ static PylibMC_Client *PylibMC_ClientType_new(PyTypeObject *, PyObject *, PyObject *); static void PylibMC_ClientType_dealloc(PylibMC_Client *); static int PylibMC_Client_init(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_get(PylibMC_Client *, PyObject *arg); static PyObject *PylibMC_Client_gets(PylibMC_Client *, PyObject *arg); static PyObject *PylibMC_Client_set(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_replace(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_add(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_prepend(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_append(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_cas(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_delete(PylibMC_Client *, PyObject *); static PyObject *PylibMC_Client_incr(PylibMC_Client *, PyObject *); static PyObject *PylibMC_Client_decr(PylibMC_Client *, PyObject *); static PyObject *PylibMC_Client_incr_multi(PylibMC_Client*, PyObject*, PyObject*); static PyObject *PylibMC_Client_get_multi(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_set_multi(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_add_multi(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_delete_multi(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_get_behaviors(PylibMC_Client *); static PyObject *PylibMC_Client_set_behaviors(PylibMC_Client *, PyObject *); static PyObject *PylibMC_Client_get_stats(PylibMC_Client *, PyObject *); static PyObject *PylibMC_Client_flush_all(PylibMC_Client *, PyObject *, PyObject *); static PyObject *PylibMC_Client_disconnect_all(PylibMC_Client *); static PyObject *PylibMC_Client_clone(PylibMC_Client *); static PyObject *PylibMC_ErrFromMemcachedWithKey(PylibMC_Client *, const char *, memcached_return, const char *, Py_ssize_t); static PyObject *PylibMC_ErrFromMemcached(PylibMC_Client *, const char *, memcached_return); static PyObject *_PylibMC_Unpickle(const char *, size_t); static PyObject *_PylibMC_Pickle(PyObject *); static int _PylibMC_CheckKey(PyObject *); static int _PylibMC_CheckKeyStringAndSize(char *, Py_ssize_t); static int _PylibMC_SerializeValue(PyObject *key_obj, PyObject *key_prefix, PyObject *value_obj, time_t time, pylibmc_mset *serialized); static void _PylibMC_FreeMset(pylibmc_mset*); static PyObject *_PylibMC_RunSetCommandSingle(PylibMC_Client *self, _PylibMC_SetCommand f, char *fname, PyObject *args, PyObject *kwds); static PyObject *_PylibMC_RunSetCommandMulti(PylibMC_Client *self, _PylibMC_SetCommand f, char *fname, PyObject *args, PyObject *kwds); static bool _PylibMC_RunSetCommand(PylibMC_Client *self, _PylibMC_SetCommand f, char *fname, pylibmc_mset *msets, size_t nkeys, size_t min_compress); static int _PylibMC_Deflate(char *value, size_t value_len, char **result, size_t *result_len); static bool _PylibMC_IncrDecr(PylibMC_Client *, pylibmc_incr *, size_t); /* }}} */ /* {{{ Type's method table */ static PyMethodDef PylibMC_ClientType_methods[] = { {"get", (PyCFunction)PylibMC_Client_get, METH_O, "Retrieve a key from a memcached."}, {"gets", (PyCFunction)PylibMC_Client_gets, METH_O, "Retrieve a key and cas_id from a memcached."}, {"set", (PyCFunction)PylibMC_Client_set, METH_VARARGS|METH_KEYWORDS, "Set a key unconditionally."}, {"replace", (PyCFunction)PylibMC_Client_replace, METH_VARARGS|METH_KEYWORDS, "Set a key only if it exists."}, {"add", (PyCFunction)PylibMC_Client_add, METH_VARARGS|METH_KEYWORDS, "Set a key only if doesn't exist."}, {"prepend", (PyCFunction)PylibMC_Client_prepend, METH_VARARGS|METH_KEYWORDS, "Prepend data to a key."}, {"append", (PyCFunction)PylibMC_Client_append, METH_VARARGS|METH_KEYWORDS, "Append data to a key."}, {"cas", (PyCFunction)PylibMC_Client_cas, METH_VARARGS|METH_KEYWORDS, "Attempt to compare-and-store a key by CAS ID."}, {"delete", (PyCFunction)PylibMC_Client_delete, METH_VARARGS, "Delete a key."}, {"incr", (PyCFunction)PylibMC_Client_incr, METH_VARARGS, "Increment a key by a delta."}, {"decr", (PyCFunction)PylibMC_Client_decr, METH_VARARGS, "Decrement a key by a delta."}, {"incr_multi", (PyCFunction)PylibMC_Client_incr_multi, METH_VARARGS|METH_KEYWORDS, "Increment more than one key by a delta."}, {"get_multi", (PyCFunction)PylibMC_Client_get_multi, METH_VARARGS|METH_KEYWORDS, "Get multiple keys at once."}, {"set_multi", (PyCFunction)PylibMC_Client_set_multi, METH_VARARGS|METH_KEYWORDS, "Set multiple keys at once."}, {"add_multi", (PyCFunction)PylibMC_Client_add_multi, METH_VARARGS|METH_KEYWORDS, "Add multiple keys at once."}, {"delete_multi", (PyCFunction)PylibMC_Client_delete_multi, METH_VARARGS|METH_KEYWORDS, "Delete multiple keys at once."}, {"get_behaviors", (PyCFunction)PylibMC_Client_get_behaviors, METH_NOARGS, "Get behaviors dict."}, {"set_behaviors", (PyCFunction)PylibMC_Client_set_behaviors, METH_O, "Set behaviors dict."}, {"get_stats", (PyCFunction)PylibMC_Client_get_stats, METH_VARARGS, "Retrieve statistics from all memcached servers."}, {"flush_all", (PyCFunction)PylibMC_Client_flush_all, METH_VARARGS|METH_KEYWORDS, "Flush all data on all servers."}, {"disconnect_all", (PyCFunction)PylibMC_Client_disconnect_all, METH_NOARGS, "Disconnect from all servers and reset own state."}, {"clone", (PyCFunction)PylibMC_Client_clone, METH_NOARGS, "Clone this client entirely such that it is safe to access from " "another thread. This creates a new connection."}, {NULL, NULL, 0, NULL} }; /* }}} */ /* {{{ Type def */ static PyTypeObject PylibMC_ClientType = { PyObject_HEAD_INIT(NULL) 0, "client", sizeof(PylibMC_Client), 0, (destructor)PylibMC_ClientType_dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, "memcached client type", 0, 0, 0, 0, 0, 0, PylibMC_ClientType_methods, 0, 0, 0, 0, 0, 0, 0, (initproc)PylibMC_Client_init, 0, (newfunc)PylibMC_ClientType_new, //PyType_GenericNew, 0, 0, 0, 0, 0, 0, 0, 0 }; /* }}} */ #endif /* def __PYLIBMC_H__ */ pylibmc-1.2.3/docs/0000755000076500000240000000000011746271113015050 5ustar lericsonstaff00000000000000pylibmc-1.2.3/docs/_themes/0000755000076500000240000000000011746271113016474 5ustar lericsonstaff00000000000000pylibmc-1.2.3/docs/_themes/sap/0000755000076500000240000000000011746271113017257 5ustar lericsonstaff00000000000000pylibmc-1.2.3/docs/_themes/sap/layout.html0000644000076500000240000000013411612254115021453 0ustar lericsonstaff00000000000000{% extends "default/layout.html" %} {%- block relbar2 %}{% endblock %} pylibmc-1.2.3/docs/_themes/sap/static/0000755000076500000240000000000011746271113020546 5ustar lericsonstaff00000000000000pylibmc-1.2.3/docs/_themes/sap/static/main.css_t0000644000076500000240000001311711702247236022533 0ustar lericsonstaff00000000000000/* * default.css_t * ~~~~~~~~~~~~~ * * Sphinx stylesheet -- default theme. * * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ @import url("default.css"); /* -- page layout ----------------------------------------------------------- */ body { font-family: {{ theme_bodyfont }}; font-size: 100%; background: {{ theme_bgcolor }} url('http://sendapatch.se/bg.png'); color: #000; margin: 0; padding: 0; } div.document { margin: 1em auto; width: 45em; background-color: {{ theme_bgcolor }}; border: 1px solid {{ theme_contentboxcolor }}; } div.documentwrapper { float: left; width: 100%; } div.bodywrapper { margin: 0 0 0 230px; } div.body { background-color: {{ theme_contentbgcolor }}; color: {{ theme_textcolor }}; padding: 0 20px 30px 20px; } {%- if theme_rightsidebar|tobool %} div.bodywrapper { margin: 0 230px 0 0; } {%- endif %} div.footer { color: {{ theme_footertextcolor }}; width: 100%; padding: 9px 0 9px 0; text-align: center; font-size: 75%; } div.footer a { color: {{ theme_footertextcolor }}; text-decoration: underline; } div.related { background-color: {{ theme_relbarbgcolor }}; line-height: 30px; color: {{ theme_relbartextcolor }}; } div.related a { color: {{ theme_relbarlinkcolor }}; } div.sphinxsidebar { {%- if theme_stickysidebar|tobool %} top: 30px; bottom: 0; margin: 0; position: fixed; overflow: auto; height: auto; {%- endif %} {%- if theme_rightsidebar|tobool %} float: right; {%- if theme_stickysidebar|tobool %} right: 0; {%- endif %} {%- endif %} } {%- if theme_stickysidebar|tobool %} /* this is nice, but it it leads to hidden headings when jumping to an anchor */ /* div.related { position: fixed; } div.documentwrapper { margin-top: 30px; } */ {%- endif %} div.sphinxsidebar h3 { font-family: {{ theme_headfont }}; color: {{ theme_sidebartextcolor }}; font-size: 1.4em; font-weight: normal; margin: 0; padding: 0; } div.sphinxsidebar h3 a { color: {{ theme_sidebartextcolor }}; } div.sphinxsidebar h4 { font-family: {{ theme_headfont }}; color: {{ theme_sidebartextcolor }}; font-size: 1.3em; font-weight: normal; margin: 5px 0 0 0; padding: 0; } div.sphinxsidebar p { color: {{ theme_sidebartextcolor }}; } div.sphinxsidebar p.topless { margin: 5px 10px 10px 10px; } div.sphinxsidebar ul { margin: 10px; padding: 0; color: {{ theme_sidebartextcolor }}; } div.sphinxsidebar a { color: {{ theme_sidebarlinkcolor }}; } div.sphinxsidebar input { border: 1px solid {{ theme_sidebarlinkcolor }}; font-family: sans-serif; font-size: 1em; } {% if theme_collapsiblesidebar|tobool %} /* for collapsible sidebar */ div#sidebarbutton { background-color: {{ theme_sidebarbtncolor }}; } {% endif %} /* -- hyperlink styles ------------------------------------------------------ */ a { color: {{ theme_linkcolor }}; text-decoration: none; } a:visited { color: {{ theme_visitedlinkcolor }}; text-decoration: none; } a:hover { text-decoration: underline; } {% if theme_externalrefs|tobool %} a.external { text-decoration: none; border-bottom: 1px dashed {{ theme_linkcolor }}; } a.external:hover { text-decoration: none; border-bottom: none; } a.external:visited { text-decoration: none; border-bottom: 1px dashed {{ theme_visitedlinkcolor }}; } {% endif %} /* -- body styles ----------------------------------------------------------- */ div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { font-family: {{ theme_headfont }}; background-color: {{ theme_headbgcolor }}; font-weight: normal; color: {{ theme_headtextcolor }}; border-bottom: 1px solid {{ theme_contentboxcolor }}; margin: 20px -20px 10px -20px; padding: 3px 0 3px 10px; } div.body h1 { margin-top: 0; font-size: 200%; } div.body h2 { font-size: 160%; } div.body h3 { font-size: 140%; } div.body h4 { font-size: 120%; } div.body h5 { font-size: 110%; } div.body h6 { font-size: 100%; } a.headerlink { color: {{ theme_headlinkcolor }}; font-size: 0.8em; padding: 0 4px 0 4px; text-decoration: none; } a.headerlink:hover { background-color: {{ theme_headlinkcolor }}; color: white; } div.body p, div.body dd, div.body li { text-align: justify; line-height: 130%; } div.admonition p.admonition-title + p { display: inline; } div.admonition p { margin-bottom: 5px; } div.admonition pre { margin-bottom: 5px; } div.admonition ul, div.admonition ol { margin-bottom: 5px; } div.note, div.seealso { background-color: transparent; border: 1px solid {{ theme_contentboxcolor }}; } div.topic { background-color: #eee; } div.warning { background-color: #ffe4e4; border: 1px solid #f66; } p.admonition-title { display: inline; } p.admonition-title:after { content: ":"; } pre { padding: 5px; background-color: {{ theme_codebgcolor }}; color: {{ theme_codetextcolor }}; line-height: 120%; border: 1px solid #ac9; border-left: none; border-right: none; } tt { background-color: #ecf0f3; padding: 0 1px 0 1px; font-size: 0.95em; } th { background-color: transparent; } .warning tt { background: #efc2c2; } .note tt { background: #d6d6d6; } .viewcode-back { font-family: {{ theme_bodyfont }}; } div.viewcode-block:target { background-color: #f4debf; border-top: 1px solid #ac9; border-bottom: 1px solid #ac9; } pylibmc-1.2.3/docs/_themes/sap/theme.conf0000644000076500000240000000137311702247236021235 0ustar lericsonstaff00000000000000[theme] inherit = default stylesheet = main.css pygments_style = sphinx [options] rightsidebar = false stickysidebar = false collapsiblesidebar = false externalrefs = false nosidebar = true bgcolor = #fefefe contentbgcolor = #efefef contentboxcolor = #c0c0c0 footertextcolor = #2e2e2e sidebarbtncolor = #3c6e83 sidebartextcolor = #ffffff sidebarlinkcolor = #98dbcc relbarbgcolor = #133f52 relbartextcolor = #ffffff relbarlinkcolor = #ffffff textcolor = #000000 headbgcolor = #f2f2f2 headtextcolor = #20435c headlinkcolor = #c60f0f linkcolor = #355f7c visitedlinkcolor = #355f7c codebgcolor = #eeffcc codetextcolor = #333333 bodyfont = 'Helvetica', Arial, sans-serif headfont = 'Helvetica', Arial, sans-serif pylibmc-1.2.3/docs/behaviors.rst0000644000076500000240000001742311746271101017570 0ustar lericsonstaff00000000000000.. _behaviors: =========== Behaviors =========== libmemcached is a lot more flexible than python-memcached, and has provisions for configuring so-called *behaviors*. :mod:`pylibmc` wraps these in a Python interface. Not all of the available behaviors make sense for Python, or are hard to make use of, and as such some behaviors have been intentionally hidden or exposed in some other way (UDP and the binary protocol are examples of this.) Generally, a behavior's value should be an integer value. The exceptions are hashing and distribution, which :mod:`pylibmc` translates with the C constants' string equivalents, for readability. Other than that, the behaviors are more or less one to one mappings of libmemcached behavior constants. .. _hash: ``"hash"`` Specifies the default hashing algorithm for keys. See Hashing_ for more information and possible values. .. _distribution: ``"distribution"`` Specifies different means of distributing values to servers. See Distribution_ for more information and possible values. .. _ketama: ``"ketama"`` Setting this behavior to ``True`` is a shortcut for setting ``"hash"`` to ``"md5"`` and ``"distribution"`` to ``"consistent ketama"``. .. _ketama_weighted: ``"ketama_weighted"`` Exactly like the ``"ketama"`` behavior, but also enables the weighting support. .. _ketama_hash: ``"ketama_hash"`` Sets the hashing algorithm for host mapping on continuum. Possible values include those for the ``"hash"`` behavior. .. _buffer_requests: ``"buffer_requests"`` Enabling buffered I/O causes commands to "buffer" instead of being sent. Any action that gets data causes this buffer to be be sent to the remote connection. Quiting the connection or closing down the connection will also cause the buffered data to be pushed to the remote connection. .. _cache_lookups: ``"cache_lookups"`` Enables the named lookups cache, so that DNS lookups are made only once. .. _no_block: ``"no_block"`` Enables asychronous I/O. This is the fastest transport available for storage functions. .. _tcp_nodelay: ``"tcp_nodelay"`` Setting this behavior will enable the ``TCP_NODELAY`` socket option, which disables Nagle's algorithm. This obviously only makes sense for TCP connections. .. _cas: ``"cas"`` Enables support for CAS operations. .. _verify_keys: ``"verify_keys"`` Setting this behavior will test if the keys for validity before sending to memcached. .. _connect_timeout: ``"connect_timeout"`` In non-blocking, mode this specifies the timeout of socket connection. .. _receive_timeout: ``"receive_timeout"`` "This sets the microsecond behavior of the socket against the SO_RCVTIMEO flag. In cases where you cannot use non-blocking IO this will allow you to still have timeouts on the reading of data." .. _send_timeout: ``"send_timeout"`` "This sets the microsecond behavior of the socket against the SO_SNDTIMEO flag. In cases where you cannot use non-blocking IO this will allow you to still have timeouts on the sending of data." .. _num_replicas: ``"num_replicas"`` Poor man's high-availability solution. Specifies numbers of replicas that should be made for a given item, on different servers. "[Replication] does not dedicate certain memcached servers to store the replicas in, but instead it will store the replicas together with all of the other objects (on the 'n' next servers specified in your server list)." .. _remove_failed: ``"remove_failed"`` Remove a server from the server list after operations on it have failed for the specified number of times in a row. See the `section on failover `. .. _failure_limit: ``"failure_limit"`` : deprecated Use ``"remove_failed"`` if at all possible. Remove a server from the server list after operations on it have failed for the specified number of times in a row. .. _auto_eject: ``"auto_eject"`` : deprecated Use ``"remove_failed"`` if at all possible. With this behavior set, hosts which have been disabled will be removed from the list of servers after ``"failure_limit"``. Hashing ------- Basically, the hasher decides how a key is mapped to a specific memcached server. The available hashers are: * ``"default"`` - libmemcached's home-grown hasher * ``"md5"`` - MD5 * ``"crc"`` - CRC32 * ``"fnv1_64"`` - 64-bit FNV-1_ * ``"fnv1a_64"`` - 64-bit FNV-1a * ``"fnv1_32"`` - 32-bit FNV-1 * ``"fnv1a_32"`` - 32-bit FNV-1a * ``"murmur"`` - MurmurHash_ If :mod:`pylibmc` was built against a libmemcached using ``--enable-hash_hsieh``, you can also use ``"hsieh"``. .. _hashing-and-python-memcached: Hashing and python-memcached ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ python-memcached up until version 1.45 used a CRC32-based hashing algorithm not reproducible by libmemcached. You can change the hasher for python-memcached using the cmemcache_hash_ module, which will make it not only compatible with cmemcache_, but also the ``"crc"`` hasher in libmemcached. python-memcached 1.45 and later incorporated ``cmemcache_hash`` as its default hasher, and so will interoperate with libmemcached provided the libmemcached clients are told to use the CRC32-style hasher. This can be done in :mod:`pylibmc` as follows:: >>> mc.behaviors["hash"] = "crc" .. _FNV-1: http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash .. _MurmurHash: http://en.wikipedia.org/wiki/MurmurHash .. _cmemcache_hash: http://pypi.python.org/pypi/cmemcache_hash .. _cmemcache: http://gijsbert.org/cmemcache/ .. _hsieh: http://www.azillionmonkeys.com/qed/hash.html Distribution ------------ When using multiple servers, there are a few takes on how to choose a server from the set of specified servers. The default method is ``"modula"``, which is what most implementations use. You can enable consistent hashing by setting distribution to ``"consistent"``. Modula-based distribution is very simple. It works by taking the hash value, modulo the length of the server list. For example, consider the key ``"foo"`` under the ``"crc"`` hasher:: >>> servers = ["a", "b", "c"] >>> crc32_hash(key) 3187 >>> 3187 % len(servers) 1 >>> servers[1] 'b' However, if one was to add a server or remove a server, every key would be displaced by one - in effect, changing your server list would more or less reset the cache. Consistent hashing solves this at the price of a more costly key-to-server lookup function, `last.fm's RJ explains how it works`__. __ http://www.last.fm/user/RJ/journal/2007/04/10/rz_libketama_-_a_consistent_hashing_algo_for_memcache_clients Failover -------- Most people desire the classical "I don't really care" type of failover support: if a server goes down, just use another one. This, sadly, is not an option with libmemcached for the time being. When libmemcached introduced a behavior called ``remove_failed``, two other behaviors were deprecated in its stead called ``auto_eject`` and ``failure_limit`` -- this new behavior is a combination of the latter two. When enabled, the numeric value is the number of times a server may fail before it is ejected, and when not, no ejection occurs. "Ejection" simply means *libmemcached will stop using that server without trying any others.* So, if you configure the behaviors ``remove_failed=4`` and ``retry_timeout=10`` and one of your four servers go down for some reason, then the first request to that server will trigger whatever actual error occurred (connection reset, read error, etc), then the subsequent requests to that server within 10 seconds will all raise ``ServerDown``, then again an actual request is made and the cycle repeats until four consequent errors have occurred, at which point ``ServerDead`` will be raised immediately. In other words, ``ServerDown`` means that if the server comes back up, it goes into rotation; ``ServerDead`` means that this key is unusable until the client is reset. pylibmc-1.2.3/docs/coders.rst0000644000076500000240000000046311737041152017062 0ustar lericsonstaff00000000000000The List of Honored Men ======================= * Ludvig Ericson * ketralnis * Ruda Moura * Noah Silas * Johan Bergström * Andrew McFague * James Brown pylibmc-1.2.3/docs/conf.py0000644000076500000240000001642411613212657016357 0ustar lericsonstaff00000000000000# -*- coding: utf-8 -*- # # pylibmc documentation build configuration file, created by # sphinx-quickstart on Mon Jun 13 13:53:22 2011. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os # Find dev-mode pylibmc pylibmc_dir = os.environ.get("PYLIBMC_DIR") if not pylibmc_dir: raise RuntimeError("please set PYLIBMC_DIR") elif not os.path.exists(pylibmc_dir): raise RuntimeError("PYLIBMC_DIR %r does not exist" % (pylibmc_dir,)) elif not os.path.isdir(pylibmc_dir): raise RuntimeError("PYLIBMC_DIR %r is not a directory" % (pylibmc_dir,)) else: sys.path.insert(0, pylibmc_dir) import pylibmc # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ["sphinx.ext.autodoc"] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = pylibmc.__name__ copyright = u'2011, Ludvig Ericson' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = pylibmc.__version__ # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'sap' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. html_theme_path = ["_themes"] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'pylibmcdoc' # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'pylibmc.tex', u'pylibmc Documentation', u'Ludvig Ericson', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'pylibmc', u'pylibmc Documentation', [u'Ludvig Ericson'], 1) ] pylibmc-1.2.3/docs/index.rst0000644000076500000240000000611411702247244016713 0ustar lericsonstaff00000000000000============================================== :mod:`pylibmc` - Python client for memcached ============================================== .. currentmodule:: pylibmc :mod:`pylibmc` is a client in Python for memcached_. It is a wrapper around TangentOrg_'s libmemcached_ library. The interface is intentionally made as close to python-memcached_ as possible, so that applications can drop-in replace it. :mod:`pylibmc` leverages among other things configurable behaviors, data pickling, data compression, battle-tested GIL retention, consistent distribution, and the binary memcached protocol. .. _TangentOrg: http://tangent.org/ .. _memcached: http://memcached.org/ .. _libmemcached: http://libmemcached.org/libMemcached.html .. _python-memcached: http://www.tummy.com/Community/software/python-memcached/ Example usage ============= Create a memcached connection and configure it:: >>> import pylibmc >>> mc = pylibmc.Client(["127.0.0.1"], binary=True, ... behaviors={"tcp_nodelay": True, ... "ketama": True}) .. hint:: In earlier versions ``behaviors`` was no keyword argument, only an attribute. To safe-guard version compatibility use ``mc.behaviors = {...}`` Basic memcached operations can be accomplished with the mapping interface:: >>> mc["some_key"] = "Some value" >>> mc["some_key"] 'Some value' >>> del mc["some_key"] >>> "some_key" in mc False "Classic" style memcached operations allow for more control and clarity:: >>> mc.set("some_key", "Some value") True >>> value = mc.get("some_key") >>> value 'Some value' >>> mc.set("another_key", 3) True >>> mc.delete("another_key") True Automatic pickling of complex Python types:: >>> mc.set("complex_plane=awesome", 4+3j) True >>> mc.get("complex_plane=awesome") (4+3j) >>> import fractions >>> mc.set("structured", {"a": ("b", "c"), ... "a2": fractions.Fraction(1, 3)}) True >>> mc.get("structured") {'a': ('b', 'c'), 'a2': Fraction(1, 3)} Atomic memcached-side increments and decrements:: >>> mc.set("key", "1") # str or int is fine True >>> mc.incr("key") 2L >>> mc.decr("key") 1L Batch operations lessen GIL contention and thus I/O is faster:: >>> mc.get_multi(["key", "another_key"]) {'key': '1'} >>> mc.set_multi({"cats": ["on acid", "furry"], "dogs": True}) [] >>> mc.get_multi(["cats", "dogs"]) {'cats': ['on acid', 'furry'], 'dogs': True} >>> mc.delete_multi(["cats", "dogs", "nonextant"]) False Further Reading =============== .. toctree:: :maxdepth: 2 install reference pooling behaviors misc Links and resources =================== * sendapatch.se: `sendapatch.se/`__ * GitHub: `github.com/lericson/pylibmc`__ * PyPI: `pypi.python.org/pypi/pylibmc`__ * libmemcached: `tangent.org/552/libmemcached.html`__ * memcached: `memcached.org/`__ __ http://sendapatch.se/ __ http://github.com/lericson/pylibmc __ http://pypi.python.org/pypi/pylibmc __ libmemcached_ __ http://memcached.org/ pylibmc-1.2.3/docs/install.rst0000644000076500000240000000160711631776714017266 0ustar lericsonstaff00000000000000============ Installing ============ Requirements ============ * Python 2.5 or later * libmemcached 0.32 or later (last test with 0.51) * zlib (required for compression support) * libsasl2 (required for authentication support) Building ======== Like any Python package, use ``setup.py``:: $ python setup.py install --with-libmemcached=/opt/local You only need to specify ``--with-libmemcached`` if libmemcached is not available on your C compiler's include and library path. This is the case if you use MacPorts or something like that. So for example, if one were to use MacPorts to install libmemcached, your libmemcached would end up in ``/opt/local``, hence ``--with-libmemcached=/opt/local``. From PyPI --------- Since ``easy_install`` doesn't support passing arguments to the setup script, you can also define environment variables:: LIBMEMCACHED=/opt/local easy_install pylibmc pylibmc-1.2.3/docs/make.bat0000644000076500000240000001064111612254115016452 0ustar lericsonstaff00000000000000@ECHO OFF REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set BUILDDIR=_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. devhelp to make HTML files and a Devhelp project echo. epub to make an epub echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages echo. changes to make an overview over all changed/added/deprecated items echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "singlehtml" ( %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pylibmc.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pylibmc.ghc goto end ) if "%1" == "devhelp" ( %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp if errorlevel 1 exit /b 1 echo. echo.Build finished. goto end ) if "%1" == "epub" ( %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub if errorlevel 1 exit /b 1 echo. echo.Build finished. The epub file is in %BUILDDIR%/epub. goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex if errorlevel 1 exit /b 1 echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "text" ( %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text if errorlevel 1 exit /b 1 echo. echo.Build finished. The text files are in %BUILDDIR%/text. goto end ) if "%1" == "man" ( %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man if errorlevel 1 exit /b 1 echo. echo.Build finished. The manual pages are in %BUILDDIR%/man. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes if errorlevel 1 exit /b 1 echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest if errorlevel 1 exit /b 1 echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) :end pylibmc-1.2.3/docs/Makefile0000644000076500000240000000675311612254115016516 0ustar lericsonstaff00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: html clean dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pylibmc.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pylibmc.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/pylibmc" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pylibmc" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." make -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." clean: -rm -rf $(BUILDDIR)/* pylibmc-1.2.3/docs/misc.rst0000644000076500000240000000574111612254115016537 0ustar lericsonstaff00000000000000=========================== Miscellaneous information =========================== In 1727, pennies featured the motto "Mind your own business!" Moreso, the average giraffe's tongue is two feet, and elephants can't stand on their heads (duh). Also, the average elevator travels 10,000 miles per year and Thomas Edison was afraid of the dark. Differences from ``python-memcached`` ===================================== In general, because :mod:`pylibmc` is built on top of libmemcached, it issues exceptions for a lot of errors which ``python-memcached`` doesn't. One example is if a memcached goes down, libmemcached will report an error whereas ``python-memcached`` will simply cycle to the next memcached. On a similar note, :mod:`pylibmc` won't raise a ``ValueError`` if one uses ``Client.inc`` on a non-existent key, instead a :attr:`pylibmc.NotFound` exception is raised. Negative timeouts are treated the same way as zero timeouts are in :mod:`pylibmc`. ``python-memcached`` treats this as immediate expiry, returning success while not setting any value. This might raise exceptions in the future. The most notable difference is the hashing. See :ref:`the hashing docs ` for information and how to resolve the issue. When setting huge key values, i.e. around 1MB, will have :mod:`pylibmc` complain loudly whereas ``python-memcached`` simply ignores the error and returns. .. _exceptions: Exceptions ========== Most of all ``libmemcached`` error numbers are translated to exception classes except where it doesn't make sense. All :mod:`pylibmc` exceptions derive from :attr:`pylibmc.Error`, so to attempt a memcached operation and never fail, something like this could be used (although it should be noted soft errors are handled via return values; errors aren't always exceptional when caching things):: try: op() except pylibmc.Error as e: log_exc() Should you for some reason need the actual error code returned from ``libmemcached``, simply sneak a peak at the ``retcode`` attribute. If you're interested in exactly what maps to what, see ``_pylibmcmodule.h``. .. warning:: Never ignore exceptional states. Any programmer worth his salt knows that an except block **must** handle the error, or make the error obvious to administrators (use logs, people!) .. _compression: Compression =========== libmemcached has no built-in support for compression, and so to be compatible with python-memcached, :mod:`pylibmc` implements it by itself. Compression requires zlib to be available when building :mod:`pylibmc`, which shouldn't be an issue for any up-to-date system. Threading ========= A single connection instance is *not* thread-safe. That is to say, you'll probably wind up dissynchronizing server and client if you use the same connection in many threads simultaneously. Instead, one is encouraged to use the existing provisions to that end, for now, see the inline code documentation (esp. ``pydoc pylibmc.ThreadMappedPool``) pylibmc-1.2.3/docs/pooling.rst0000644000076500000240000000760111613225707017256 0ustar lericsonstaff00000000000000.. currentmodule:: pylibmc.pools ========= Pooling ========= :Author: Ludvig Ericson :See also: `Pooling with pylibmc`__ (this document, first revision) :See also: `Pooling with pylibmc pt. 2`__ (follow-up) __ http://lericson.blogg.se/code/2009/september/draft-sept-20-2009.html __ http://lericson.blogg.se/code/2009/september/pooling-with-pylibmc-pt-2.html .. note:: This was originally a blog post. Edited and provided here for your convenience. I was discussing how to implement pooling for :mod:`pylibmc` when I realized what `libmemcachedutil`'s pooling is - or rather, what it isn't. It's not a magical solution for concurrently doing anything at all, it's not anything like that -- it just helps you with thread-safety. In Python, however, we've got the global interpreter lock, the GIL. This lock must always be held by the thread that is dealing with anything Python. The Python interpreter itself isn't thread-safe, or rather, it is with the GIL. This means that whenever Python code is running, you'll be sure to have exclusive access to all of Python's memory (unless something is misbehaving.) In turn, this means that the usecase for using `libmemcachedutil` in a Python library is rather slim. An example with Werkzeug ======================== This is a Werkzeug-based WSGI application which would be run in multiple threads concurrently and still not have issues with races: .. code-block:: python # Configuration n_threads = 12 mc_addrs = "10.0.1.1", "10.0.1.2", "10.0.1.3" mc_pool_size = n_threads # Application import pylibmc from contextlib import contextmanager from pprint import pformat from werkzeug.wrappers import Request, Response from werkzeug.exceptions import NotFound class ClientPool(list): @contextmanager def reserve(self): mc = self.pop() try: yield mc finally: self.append(mc) mc = pylibmc.Client(mc_addrs) mc_pool = ClientPool(mc.clone() for i in xrange(mc_pool_size)) @Request.application def my_app(request): with mc_pool.reserve() as mc: key = request.path[1:].encode("ascii") val = mc.get(key) if not val: return NotFound(key) return Response(pformat(val)) if __name__ == "__main__": from werkzeug.serving import run_simple run_simple("0.0.0.0", 5050, my_app) It's fully-functional example of how one could implement pooling with :mod:`pylibmc`, and very much so in the same way that people do with `libmemcachedutil`. Paste it into a script file, it runs out of the box. FIFO-like pooling ================= The aforementioned type of pool is already implemented in :mod:`pylibmc` as :class:`pylibmc.ClientPool`, with a couple of other bells & whistles as well as tests (hint: don't implement it yourself.) Its documentation speaks for itself: .. autoclass:: pylibmc.ClientPool .. automethod:: fill .. automethod:: reserve The use is identical to what was demonstrated above, apart from initialization, that would look like this: .. code-block:: python mc = pylibmc.Client(mc_addrs) mc_pool = pylibmc.ClientPool(mc, mc_pool_size) Thread-mapped pooling ===================== Another possibility is to have a data structure that remembers the thread name (i.e. key it by thread ID or so.) Each thread would reserve its client in the dict on each request. If none exists, it would clone a master instance. Again, the documentation: .. autoclass:: pylibmc.ThreadMappedPool .. automethod:: reserve .. automethod:: relinquish A note on relinquishing ----------------------- You must be sure to call :meth:`ThreadMappedPool.relinquish` *before* exiting a thread that has used the pool, *from that thread*! Otherwise, some clients will never be reclaimed and you will have stale, useless connections. pylibmc-1.2.3/docs/reference.rst0000644000076500000240000001340611613025037017537 0ustar lericsonstaff00000000000000=========== Reference =========== .. class:: pylibmc.Client(servers[, binary=False, behaviors=None]) Interface to a set of memcached servers. *servers* is a sequence of strings specifying the servers to use. *binary* specifies whether or not to use the binary protocol to talk to the memcached servers. *behaviors*, if given, is passed to :meth:`Client.set_behaviors` after initialization. Supported transport mechanisms are TCP, UDP and UNIX domain sockets. The default transport type is TCP. To specify UDP, the server address should be prefixed with ``"udp:"``, as in ``"udp:127.0.0.1"``. To specify UNIX domain socket, the server address must contain a slash, as in ``"./foo.sock"``. Mixing transport types is prohibited by :mod:`pylibmc` as this is not supported by libmemcached. .. method:: clone() -> clone Clone client, making new connections as necessary. .. Reading .. method:: get(key) -> value Get *key* if it exists, otherwise ``None``. .. method:: get_multi(keys[, key_prefix=None]) -> values Get each of the keys in sequence *keys*. If *key_prefix* is given, specifies a string to prefix each of the values in *keys* with. Returns a mapping of each unprefixed key to its corresponding value in memcached. If a key doesn't exist, no corresponding key is set in the returned mapping. .. Writing .. method:: set(key, value[, time=0, min_compress_len=0]) -> success Set *key* to *value*. :param key: Key to use :param value: Value to set :param time: Time until expiry :param min_compress_len: Minimum length before compression is triggered If *time* is given, it specifies the number of seconds until *key* will expire. Default behavior is to never expire (equivalent of specifying ``0``). If *min_compress_len* is given, it specifies the maximum number of actual bytes stored in memcached before compression is used. Default behavior is to never compress (which is what ``0`` means). See :ref:`compression`. .. method:: set_multi(mapping, [, time=0, key_prefix=None]) -> failed_keys Set multiple keys as given by *mapping*. If *key_prefix* is specified, each of the keys in *mapping* is prepended with this value. Returns a list of keys which were not set for one reason or another, without their optional key prefix. .. method:: add(key, value[, time, min_compress_len]) -> success Sets *key* if it does not exist. .. seealso:: :meth:`set`, :meth:`replace` .. method:: replace(key, value[, time, min_compress_len]) -> success Sets *key* only if it already exists. .. seealso:: :meth:`set`, :meth:`add` .. method:: append(key, value) -> success Append *value* to *key* (i.e., ``m[k] = m[k] + v``). .. note:: Uses memcached's appending support, and therefore should never be used on keys which may be compressed or non-string values. .. method:: prepend(key, value) -> success Prepend *value* to *key* (i.e., ``m[k] = v + m[k]``). .. note:: Uses memcached's prepending support, and therefore should never be used on keys which may be compressed or non-string values. .. method:: incr(key[, delta=1]) -> value Increment value at *key* by *delta*. Returns the new value for *key*, after incrementing. Works for both strings and integer types. .. note:: There is currently no way to set a default for *key* when incrementing. .. method:: decr(key[, delta=1]) -> value Decrement value at *key* by *delta*. Returns the new value for *key*, after decrementing. Works for both strings and integer types, but will never decrement below zero. .. note:: There is currently no way to set a default for *key* when decrementing. .. Deleting .. method:: delete(key[, time=0]) -> deleted Delete *key* if it exists. If *time* is non-zero, this is equivalent of setting an expiry time for a key, i.e., the key will cease to exist after that many seconds. Returns ``True`` if the key was deleted, ``False`` otherwise (as is the case if it wasn't set in the first place.) .. note:: Some versions of libmemcached are unable to set *time* for a delete. This is true of versions up until at least 0.38. .. method:: delete_multi(keys[, time=0, key_prefix=None]) -> deleted Delete each of key in the sequence *keys*. :param keys: Sequence of keys to delete :param time: Number of seconds until the keys are deleted :param key_prefix: Prefix for the keys to delete If *time* is zero, the keys are deleted immediately. Returns ``True`` if all keys were successfully deleted, ``False`` otherwise (as is the case if it wasn't set in the first place.) .. Utilities .. method:: disconnect_all() Disconnect from all servers and reset internal state. Exposed mainly for compatibility with python-memcached, as there really is no logical reason to do this. .. method:: flush_all() -> success Flush all data from all servers. .. note:: This clears the specified memcacheds fully and entirely. .. method:: get_stats() -> [(name, stats), ...] Retrieve statistics from each of the connected memcached instances. Returns a list of two-tuples of the format ``(name, stats)``. *stats* is a mapping of statistics item names to their values. Whether or not a key exists depends on the version of libmemcached and memcached used. .. data:: behaviors The behaviors used by the underlying libmemcached object. See :ref:`behaviors` for more information. pylibmc-1.2.3/LICENSE0000644000076500000240000000272111557524262015135 0ustar lericsonstaff00000000000000Copyright (c) 2008, Ludvig Ericson All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the name of the author nor the names of the contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. pylibmc-1.2.3/MANIFEST.in0000644000076500000240000000045611652532103015656 0ustar lericsonstaff00000000000000include README.rst LICENSE MANIFEST.in setup.py setup.cfg runtests.py include _pylibmcmodule.c _pylibmcmodule.h pylibmc-version.h recursive-include pylibmc *.py recursive-include tests *.py include docs/Makefile docs/make.bat docs/conf.py recursive-include docs *.rst recursive-include docs/_themes * pylibmc-1.2.3/PKG-INFO0000644000076500000240000001210511746271113015214 0ustar lericsonstaff00000000000000Metadata-Version: 1.0 Name: pylibmc Version: 1.2.3 Summary: Quick and small memcached client for Python Home-page: http://sendapatch.se/projects/pylibmc/ Author: Ludvig Ericson Author-email: ludvig@lericson.se License: 3-clause BSD Description: `pylibmc` is a Python client for `memcached `_ written in C. See `the documentation at sendapatch.se/projects/pylibmc/`__ for more information. __ http://sendapatch.se/projects/pylibmc/ Contact ======= - Website: `sendapatch.se/ `_ - Github: `github.com/lericson `_ - IRC: ``lericson`` in ``#sendapatch`` on ``chat.freenode.net`` - E-mail: ``ludvig`` at ``sendapatch.se`` Change Log ========== New in version 1.2.0 -------------------- This release is for the people behind `reddit.com`__, for helping push development forward. Keep doing your thing. __ http://code.reddit.com/ - `sevmer.org`__ versioning scheme - Fixed GIL issues - Added CAS support (ketralnis) - Added SASL authentication (Remoun) - Added more detail to errors (spladug) - Added mapping-like behavior for clients - Fixed build errors on Mac OS X - Moved to nose__ for testing - Added ``auto_eject`` behavior - Added ``num_replicas`` behavior - Added ``remove_failed`` behavior - Removed ``cache_lookups`` behavior - Improved repr of clients (noah256) - Improved IPv6 support (JshWright) - Improved pooling behavior so it doesn't cause lock-ups - Improved tests and testing foundation - Improved documentation and structure - Internalized Sphinx documentation - Bunch of other stuff __ http://semver.org/ __ http://somethingaboutorange.com/mrl/projects/nose/ New in version 1.1 ------------------ - Removed deprecated space-based behavior names. - Acquire and release the GIL properly, thanks ketralnis__ - Add support for ``libmemcached 0.40`` - Included a more useful command-line interface - Fixed handling of NUL-byte keys in ``get_multi`` in binary protocol - Fixed some valgrind-reported memory warnings - Fixed bogus usage of time argument for delete. - 1.1.1: Fixed tests under Python 2.5 __ http://www.ketralnis.com/ New in version 1.0 ------------------ - Lots of documentation fixes and other nice things like that. - Nailed what appears to be the last outstanding memory leak. - Explicitly require libmemcached 0.32 or newer. New in version 0.9 ------------------ - Added a ``get_stats`` method, which behaves exactly like `python-memcached`'s equivalent. - Gives the empty string for empty memcached values like `python-memcached` does. - Added exceptions for most `libmemcached` return codes. - Fixed an issue with ``Client.behaviors.update``. New in version 0.8 ------------------ - Pooling helpers are now available. See ``pooling.rst`` in the distribution. - The binary protocol is now properly exposed, simply pass ``binary=True`` to the constructor and there you go. - Call signatures now match `libmemcached` 0.32, but should work with older versions. Remember to run the tests! New in version 0.7 ------------------ - Restructured some of the code, which should yield better performance (if not for that, it reads better.) - Fixed some memory leaks. - Integrated changes from `amix.dk`, which should make pylibmc work under Snow Leopard. - Add support for the boolean datatype. - Improved test-runner -- now tests ``build/lib.*/_pylibmc.so`` if available, and reports some version information. - Support for x86_64 should now work completely. - Builds with Python 2.4, tests run fine, but not officially supported. - Fixed critical bugs in behavior manipulation. New in version 0.6 ------------------ - Added compatibility with `libmemcached` 0.26, WRT error return codes. - Added `flush_all` and `disconnect_all` methods. - Now using the latest pickling protocol. New in version 0.5 ------------------ - Fixed lots of memory leaks, and added support for `libmemcached` 0.23. - Also made the code tighter in terms of compiler pedantics. New in version 0.4 ------------------ - Renamed the C module to `_pylibmc`, and added lots of `libmemcached` constants to it, as well as implemented behaviors. Platform: UNKNOWN pylibmc-1.2.3/pylibmc/0000755000076500000240000000000011746271113015557 5ustar lericsonstaff00000000000000pylibmc-1.2.3/pylibmc/__init__.py0000644000076500000240000000412711613240353017667 0ustar lericsonstaff00000000000000"""Snappy libmemcached wrapper pylibmc is a Python wrapper around TangentOrg's libmemcached library. The interface is intentionally made as close to python-memcached as possible, so that applications can drop-in replace it. Example usage ============= Create a connection and configure it:: >>> import pylibmc >>> m = pylibmc.Client(["10.0.0.1"], binary=True) >>> m.behaviors = {"tcp_nodelay": True, "ketama": True} Nevermind this doctest shim:: >>> from pylibmc.test import make_test_client >>> mc = make_test_client(behaviors=m.behaviors) Basic operation:: >>> mc.set("some_key", "Some value") True >>> value = mc.get("some_key") >>> value 'Some value' >>> mc.set("another_key", 3) True >>> mc.delete("another_key") True >>> mc.set("key", "1") # str or int is fine True Atomic increments and decrements:: >>> mc.incr("key") 2L >>> mc.decr("key") 1L Batch operation:: >>> mc.get_multi(["key", "another_key"]) {'key': '1'} >>> mc.set_multi({"cats": ["on acid", "furry"], "dogs": True}) [] >>> mc.get_multi(["cats", "dogs"]) {'cats': ['on acid', 'furry'], 'dogs': True} >>> mc.delete_multi(["cats", "dogs", "nonextant"]) False >>> mc.add_multi({"cats": ["on acid", "furry"], "dogs": True}) [] >>> mc.get_multi(["cats", "dogs"]) {'cats': ['on acid', 'furry'], 'dogs': True} >>> mc.add_multi({"cats": "not set", "dogs": "definitely not set", "bacon": "yummy"}) ['cats', 'dogs'] >>> mc.get_multi(["cats", "dogs", "bacon"]) {'cats': ['on acid', 'furry'], 'bacon': 'yummy', 'dogs': True} >>> mc.delete_multi(["cats", "dogs", "bacon"]) True Further Reading =============== See http://sendapatch.se/projects/pylibmc/ """ import _pylibmc from .consts import hashers, distributions from .client import Client from .pools import ClientPool, ThreadMappedPool support_compression = _pylibmc.support_compression support_sasl = _pylibmc.support_sasl __version__ = _pylibmc.__version__ __all__ = ["hashers", "distributions", "Client", "ClientPool", "ThreadMappedPool"] pylibmc-1.2.3/pylibmc/__main__.py0000644000076500000240000000244011612254113017642 0ustar lericsonstaff00000000000000"""Interactive shell""" import sys import code import random import pylibmc tips = [ "Want to use 127.0.0.1? Just hit Enter immediately.", "This was supposed to be a list of tips but I...", "I don't really know what to write here.", "Really, hit Enter immediately and you'll connect to 127.0.0.1.", "Did you know there's a --binary flag? Try it!", "Want to use binary mode? Pass --binary as a sole argument." ] def print_header(outf=sys.stdout): outf.write("pylibmc interactive shell\n\n") outf.write("Input list of servers, terminating by a blank line.\n") outf.write(random.choice(tips) + "\n") def collect_servers(): in_addr = raw_input("Address [127.0.0.1]: ") if in_addr: while in_addr: yield in_addr in_addr = raw_input("Address []: ") else: yield "127.0.0.1" banner = "\nmc client available as `mc`\n" def interact(servers, banner=banner, binary=False): mc = pylibmc.Client(servers, binary=binary) local = {"pylibmc": pylibmc, "mc": mc} code.interact(banner=banner, local=local) def main(): binary = False if sys.argv[1:] == ["--binary"]: binary = True print_header() interact(list(collect_servers()), binary=binary) if __name__ == "__main__": main() pylibmc-1.2.3/pylibmc/client.py0000644000076500000240000001441511746271101017411 0ustar lericsonstaff00000000000000"""Python-level wrapper client""" import _pylibmc from .consts import (hashers, distributions, all_behaviors, hashers_rvs, distributions_rvs, BehaviorDict) _all_behaviors_set = set(all_behaviors) server_type_map = {"tcp": _pylibmc.server_type_tcp, "udp": _pylibmc.server_type_udp, "unix": _pylibmc.server_type_unix} def _split_spec_type(spec): if spec.startswith("/"): return ("unix", spec) else: if spec.startswith("udp:"): return ("udp", spec[4:]) else: return ("tcp", spec) def _splitport(spec, port=None): if spec.startswith("["): if "]" not in spec: raise ValueError(spec) if spec.endswith("]"): addr = spec[1:-1] else: (addr, port) = spec[1:].rsplit("]:", 1) elif ":" in spec: (addr, port) = spec.split(":", 1) else: addr = spec return (addr, port) def translate_server_spec(server, port=11211): """Translate/normalize specification *server* into three-tuple (tp, addr, port). This is format is used by the extension module. >>> translate_server_spec("127.0.0.1") (1, '127.0.0.1', 11211) >>> translate_server_spec("udp:127.0.0.1") (2, '127.0.0.1', 11211) >>> translate_server_spec("/var/run/memcached.sock") (4, '/var/run/memcached.sock', 0) >>> translate_server_spec("127.0.0.1:22122") (1, '127.0.0.1', 22122) >>> translate_server_spec("127.0.0.1", port=1234) (1, '127.0.0.1', 1234) >>> translate_server_spec("[::1]:22122") (1, '::1', 22122) >>> translate_server_spec("[::1]") (1, '::1', 11211) """ (scheme, spec) = _split_spec_type(server) stype = server_type_map[scheme] if scheme == "unix": (addr, port) = (spec, 0) elif scheme in ("tcp", "udp"): (addr, port) = _splitport(spec, port=port) return (stype, str(addr), int(port)) def translate_server_specs(servers): addr_tups = [] for server in servers: # Anti-pattern for convenience if isinstance(server, tuple) and len(server) == 3: addr_tup = server else: addr_tup = translate_server_spec(server) addr_tups.append(addr_tup) return addr_tups class Client(_pylibmc.client): def __init__(self, servers, behaviors=None, binary=False, username=None, password=None): """Initialize a memcached client instance. This connects to the servers in *servers*, which will default to being TCP servers. If it looks like a filesystem path, a UNIX socket. If prefixed with `udp:`, a UDP connection. If *binary* is True, the binary memcached protocol is used. SASL authentication is supported if libmemcached supports it (check *pylibmc.support_sasl*). Requires both username and password. Note that SASL requires *binary*=True. """ self.binary = binary self.addresses = list(servers) super(Client, self).__init__(servers=translate_server_specs(servers), binary=binary, username=username, password=password) if behaviors is not None: self.set_behaviors(behaviors) def __repr__(self): return "%s(%r, binary=%r)" % (self.__class__.__name__, self.addresses, self.binary) def __str__(self): addrs = ", ".join(map(str, self.addresses)) return "<%s for %s, binary=%r>" % (self.__class__.__name__, addrs, self.binary) # {{{ Mapping interface def __getitem__(self, key): value = self.get(key) if value is None: raise KeyError(key) else: return value def __setitem__(self, key, value): if not self.set(key, value): raise KeyError("failed setting %r" % (key,)) def __delitem__(self, key): if not self.delete(key): raise KeyError(key) def __contains__(self, key): return self.get(key) is not None # }}} # {{{ Behaviors def get_behaviors(self): """Gets the behaviors from the underlying C client instance. Reverses the integer constants for `hash` and `distribution` into more understandable string values. See *set_behaviors* for info. """ bvrs = super(Client, self).get_behaviors() bvrs["hash"] = hashers_rvs[bvrs["hash"]] bvrs["distribution"] = distributions_rvs[bvrs["distribution"]] return BehaviorDict(self, bvrs) def set_behaviors(self, behaviors): """Sets the behaviors on the underlying C client instance. Takes care of morphing the `hash` key, if specified, into the corresponding integer constant (which the C client expects.) If, however, an unknown value is specified, it's passed on to the C client (where it most surely will error out.) This also happens for `distribution`. Translates old underscored behavior names to new ones for API leniency. """ behaviors = behaviors.copy() if "_retry_timeout" in behaviors: behaviors.setdefault("retry_timeout", behaviors.pop("_retry_timeout")) unknown = set(behaviors).difference(_all_behaviors_set) if unknown: names = ", ".join(map(str, sorted(unknown))) raise ValueError("unknown behavior names: %s" % (names,)) if behaviors.get("hash") is not None: behaviors["hash"] = hashers[behaviors["hash"]] if behaviors.get("ketama_hash") in hashers: behaviors["ketama_hash"] = hashers[behaviors["ketama_hash"]] if behaviors.get("distribution") is not None: behaviors["distribution"] = distributions[behaviors["distribution"]] return super(Client, self).set_behaviors(behaviors) behaviors = property(get_behaviors, set_behaviors) @property def behaviours(self): raise AttributeError("nobody uses british spellings") # }}} def clone(self): obj = super(Client, self).clone() obj.addresses = list(self.addresses) obj.binary = self.binary return obj if __name__ == "__main__": import doctest doctest.testmod() pylibmc-1.2.3/pylibmc/consts.py0000644000076500000240000000266511624750721017455 0ustar lericsonstaff00000000000000"""Constants and functionality related to them""" import _pylibmc #: Mapping of exception name => class errors = tuple(e for (n, e) in _pylibmc.exceptions) # *Cough* Uhm, not the prettiest of things but this unpacks all exception # objects and sets them on the package module object currently constructed. import sys modpkg = sys.modules[__name__.split(".", 1)[0]] modself = sys.modules[__name__] for name, exc in _pylibmc.exceptions: setattr(modself, name, exc) setattr(modpkg, name, exc) all_behaviors = _pylibmc.all_behaviors hashers, hashers_rvs = {}, {} distributions, distributions_rvs = {}, {} # Not the prettiest way of doing things, but works well. for name in dir(_pylibmc): if name.startswith("hash_"): key, value = name[5:], getattr(_pylibmc, name) hashers[key] = value hashers_rvs[value] = key elif name.startswith("distribution_"): key, value = name[13:].replace("_", " "), getattr(_pylibmc, name) distributions[key] = value distributions_rvs[value] = key class BehaviorDict(dict): def __init__(self, client, *args, **kwds): super(BehaviorDict, self).__init__(*args, **kwds) self.client = client def __setitem__(self, name, value): super(BehaviorDict, self).__setitem__(name, value) self.client.set_behaviors({name: value}) def update(self, d): super(BehaviorDict, self).update(d) self.client.set_behaviors(d.copy()) pylibmc-1.2.3/pylibmc/pools.py0000644000076500000240000000637411613240353017272 0ustar lericsonstaff00000000000000"""Pooling""" from __future__ import with_statement from contextlib import contextmanager from Queue import Queue class ClientPool(Queue): """Client pooling helper. This is mostly useful in threaded environments, because a client isn't thread-safe at all. Instead, what you want to do is have each thread use its own client, but you don't want to reconnect these all the time. The solution is a pool, and this class is a helper for that. >>> from pylibmc.test import make_test_client >>> mc = make_test_client() >>> pool = ClientPool() >>> pool.fill(mc, 4) >>> with pool.reserve() as mc: ... mc.set("hi", "ho") ... mc.delete("hi") ... True True """ def __init__(self, mc=None, n_slots=None): Queue.__init__(self, n_slots) if mc is not None: self.fill(mc, n_slots) @contextmanager def reserve(self, block=False): """Context manager for reserving a client from the pool. If *block* is given and the pool is exhausted, the pool waits for another thread to fill it before returning. """ mc = self.get(block) try: yield mc finally: self.put(mc) def fill(self, mc, n_slots): """Fill *n_slots* of the pool with clones of *mc*.""" for i in xrange(n_slots): self.put(mc.clone()) class ThreadMappedPool(dict): """Much like the *ClientPool*, helps you with pooling. In a threaded environment, you'd most likely want to have a client per thread. And there'd be no harm in one thread keeping the same client at all times. So, why not map threads to clients? That's what this class does. If a client is reserved, this class checks for a key based on the current thread, and if none exists, clones the master client and inserts that key. Of course this requires that you let the pool know when a thread is done with its reserved instance, so therefore ``relinquish`` must be called before thread exit. >>> from pylibmc.test import make_test_client >>> mc = make_test_client() >>> pool = ThreadMappedPool(mc) >>> with pool.reserve() as mc: ... mc.set("hi", "ho") ... mc.delete("hi") ... True True """ def __new__(cls, master): return super(ThreadMappedPool, cls).__new__(cls) def __init__(self, master): self.master = master @property def current_key(self): return thread.get_ident() @contextmanager def reserve(self): """Reserve a client. Creates a new client based on the master client if none exists for the current thread. """ key = self.current_key mc = self.pop(key, None) if mc is None: mc = self.master.clone() try: yield mc finally: self[key] = mc def relinquish(self): """Relinquish any reserved client for the current context. Call this method before exiting a thread if it might potentially use this pool. """ return self.pop(self.current_key, None) # This makes sure ThreadMappedPool doesn't exist with non-thread Pythons. try: import thread except ImportError: ThreadMappedPool = None pylibmc-1.2.3/pylibmc/test.py0000644000076500000240000000402411613240353017103 0ustar lericsonstaff00000000000000"""Test utils.""" import os import socket import _pylibmc, pylibmc from textwrap import dedent class NotAliveError(Exception): template = dedent(""" test memcached %s:%s not alive consider setting MEMCACHED_HOST and/or MEMCACHED_PORT to something more appropriate.""").lstrip() def __init__(self, svr): (self.server_type, self.address, self.port) = svr def __str__(self): return self.template % (self.address, self.port) def get_version(addr): (type_, host, port) = addr if (type_ != _pylibmc.server_type_tcp): raise NotImplementedError("test server can only be on tcp for now") else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((host, port)) s.send("version\r\n") version = s.recv(4096) finally: s.close() if not version.startswith("VERSION ") or not version.endswith("\r\n"): raise ValueError("unexpected version return: %r" % (version,)) else: version = version[8:-2] return version def is_alive(addr): try: version = get_version(addr) except (ValueError, socket.error): version = None return (bool(version), version) def make_test_client(cls=pylibmc.Client, env=None, server_type="tcp", host="127.0.0.1", port=11211, **kwds): """Make a test client. `env` overrides keyword arguments. >>> make_test_client() is not None True """ server_types = {"tcp": _pylibmc.server_type_tcp, "udp": _pylibmc.server_type_udp, "unix": _pylibmc.server_type_unix} if env is None: env = os.environ styp = env.get("MEMCACHED_TYPE", server_type) host = env.get("MEMCACHED_HOST", host) port = env.get("MEMCACHED_PORT", port) test_server = (server_types[styp], str(host), int(port)) alive, version = is_alive(test_server) if alive: return cls([test_server], **kwds) else: raise NotAliveError(test_server) pylibmc-1.2.3/pylibmc-version.h0000644000076500000240000000004011746271101017402 0ustar lericsonstaff00000000000000#define PYLIBMC_VERSION "1.2.3" pylibmc-1.2.3/README.rst0000644000076500000240000000750011653006337015612 0ustar lericsonstaff00000000000000`pylibmc` is a Python client for `memcached `_ written in C. See `the documentation at sendapatch.se/projects/pylibmc/`__ for more information. __ http://sendapatch.se/projects/pylibmc/ Contact ======= - Website: `sendapatch.se/ `_ - Github: `github.com/lericson `_ - IRC: ``lericson`` in ``#sendapatch`` on ``chat.freenode.net`` - E-mail: ``ludvig`` at ``sendapatch.se`` Change Log ========== New in version 1.2.0 -------------------- This release is for the people behind `reddit.com`__, for helping push development forward. Keep doing your thing. __ http://code.reddit.com/ - `sevmer.org`__ versioning scheme - Fixed GIL issues - Added CAS support (ketralnis) - Added SASL authentication (Remoun) - Added more detail to errors (spladug) - Added mapping-like behavior for clients - Fixed build errors on Mac OS X - Moved to nose__ for testing - Added ``auto_eject`` behavior - Added ``num_replicas`` behavior - Added ``remove_failed`` behavior - Removed ``cache_lookups`` behavior - Improved repr of clients (noah256) - Improved IPv6 support (JshWright) - Improved pooling behavior so it doesn't cause lock-ups - Improved tests and testing foundation - Improved documentation and structure - Internalized Sphinx documentation - Bunch of other stuff __ http://semver.org/ __ http://somethingaboutorange.com/mrl/projects/nose/ New in version 1.1 ------------------ - Removed deprecated space-based behavior names. - Acquire and release the GIL properly, thanks ketralnis__ - Add support for ``libmemcached 0.40`` - Included a more useful command-line interface - Fixed handling of NUL-byte keys in ``get_multi`` in binary protocol - Fixed some valgrind-reported memory warnings - Fixed bogus usage of time argument for delete. - 1.1.1: Fixed tests under Python 2.5 __ http://www.ketralnis.com/ New in version 1.0 ------------------ - Lots of documentation fixes and other nice things like that. - Nailed what appears to be the last outstanding memory leak. - Explicitly require libmemcached 0.32 or newer. New in version 0.9 ------------------ - Added a ``get_stats`` method, which behaves exactly like `python-memcached`'s equivalent. - Gives the empty string for empty memcached values like `python-memcached` does. - Added exceptions for most `libmemcached` return codes. - Fixed an issue with ``Client.behaviors.update``. New in version 0.8 ------------------ - Pooling helpers are now available. See ``pooling.rst`` in the distribution. - The binary protocol is now properly exposed, simply pass ``binary=True`` to the constructor and there you go. - Call signatures now match `libmemcached` 0.32, but should work with older versions. Remember to run the tests! New in version 0.7 ------------------ - Restructured some of the code, which should yield better performance (if not for that, it reads better.) - Fixed some memory leaks. - Integrated changes from `amix.dk`, which should make pylibmc work under Snow Leopard. - Add support for the boolean datatype. - Improved test-runner -- now tests ``build/lib.*/_pylibmc.so`` if available, and reports some version information. - Support for x86_64 should now work completely. - Builds with Python 2.4, tests run fine, but not officially supported. - Fixed critical bugs in behavior manipulation. New in version 0.6 ------------------ - Added compatibility with `libmemcached` 0.26, WRT error return codes. - Added `flush_all` and `disconnect_all` methods. - Now using the latest pickling protocol. New in version 0.5 ------------------ - Fixed lots of memory leaks, and added support for `libmemcached` 0.23. - Also made the code tighter in terms of compiler pedantics. New in version 0.4 ------------------ - Renamed the C module to `_pylibmc`, and added lots of `libmemcached` constants to it, as well as implemented behaviors. pylibmc-1.2.3/runtests.py0000755000076500000240000000261311612304062016356 0ustar lericsonstaff00000000000000#!/usr/bin/env python """Run nosetests with build/lib.* in sys.path""" # Fix up sys.path so as to include the correct build/lib.*/ directory. import sys from distutils.dist import Distribution from distutils.command.build import build build_cmd = build(Distribution({"ext_modules": True})) build_cmd.finalize_options() lib_dirn = build_cmd.build_lib sys.path.insert(0, lib_dirn) # Dump info plugin stuff import nose import logging from nose.plugins import Plugin logger = logging.getLogger("nose.plugins.pylibmc") def dump_infos(): logger.info("injected path: %s", lib_dirn) import pylibmc, _pylibmc if hasattr(_pylibmc, "__file__"): logger.info("loaded _pylibmc from %s", _pylibmc.__file__) if not _pylibmc.__file__.startswith(lib_dirn): logger.warn("double-check the source path") else: logger.warn("static _pylibmc: %s", _pylibmc) logger.info("libmemcached version: %s", _pylibmc.libmemcached_version) logger.info("pylibmc version: %s", _pylibmc.__version__) logger.info("support compression: %s", _pylibmc.support_compression) logger.info("support sasl auth: %s", _pylibmc.support_sasl) class PylibmcVersionDumper(Plugin): name = "info" enableOpt = "info" def __init__(self): Plugin.__init__(self) def begin(self): dump_infos() if __name__ == "__main__": nose.main(addplugins=[PylibmcVersionDumper()]) pylibmc-1.2.3/setup.cfg0000644000076500000240000000013011652521410015725 0ustar lericsonstaff00000000000000[nosetests] with-info=1 with-doctest=1 debug=nose.plugins.pylibmc doctest-extension=txt pylibmc-1.2.3/setup.py0000644000076500000240000000652011652520323015631 0ustar lericsonstaff00000000000000import os import sys from distutils.core import setup, Extension ## Command-line argument parsing # --with-zlib: use zlib for compressing and decompressing # --without-zlib: ^ negated # --with-zlib=: path to zlib if needed # --with-libmemcached=: path to libmemcached package if needed cmd = None use_zlib = True pkgdirs = [] # incdirs and libdirs get these libs = ["memcached"] defs = [] incdirs = [] libdirs = [] def append_env(L, e): v = os.environ.get(e) if v and os.path.exists(v): L.append(v) append_env(pkgdirs, "LIBMEMCACHED") append_env(pkgdirs, "ZLIB") # Hack up sys.argv, yay unprocessed = [] for arg in sys.argv[1:]: if arg == "--with-zlib": use_zlib = True continue elif arg == "--without-zlib": use_zlib = False continue elif arg == "--with-sasl2": libs.append("sasl2") continue elif arg == "--gen-setup": cmd = arg[2:] elif "=" in arg: if arg.startswith("--with-libmemcached=") or \ arg.startswith("--with-zlib="): pkgdirs.append(arg.split("=", 1)[1]) continue unprocessed.append(arg) sys.argv[1:] = unprocessed for pkgdir in pkgdirs: incdirs.append(os.path.join(pkgdir, "include")) libdirs.append(os.path.join(pkgdir, "lib")) if use_zlib: libs.append("z") defs.append(("USE_ZLIB", None)) ## OS X non-PPC workaround # Apple OS X 10.6 with Xcode 4 have Python compiled with PPC but they removed # support for compiling with that arch, so we have to override ARCHFLAGS. if sys.platform == "darwin" and not os.environ.get("ARCHFLAGS"): compiler_dirn = "/usr/libexec/gcc/darwin" if os.path.exists(compiler_dirn): dir_items = os.listdir(compiler_dirn) if "ppc" not in dir_items: print >>sys.stderr, "enabling osx-specific ARCHFLAGS/ppc hack" os.environ["ARCHFLAGS"] = "-arch i386 -arch x86_64" # There's a bug in