pymongo-3.6.1/0000755000076600000240000000000013246104133013430 5ustar shanestaff00000000000000pymongo-3.6.1/bson/0000755000076600000240000000000013246104133014371 5ustar shanestaff00000000000000pymongo-3.6.1/bson/binary.py0000644000076600000240000001612713245621354016246 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from uuid import UUID from bson.py3compat import PY3 """Tools for representing BSON binary data. """ BINARY_SUBTYPE = 0 """BSON binary subtype for binary data. This is the default subtype for binary data. """ FUNCTION_SUBTYPE = 1 """BSON binary subtype for functions. """ OLD_BINARY_SUBTYPE = 2 """Old BSON binary subtype for binary data. This is the old default subtype, the current default is :data:`BINARY_SUBTYPE`. """ OLD_UUID_SUBTYPE = 3 """Old BSON binary subtype for a UUID. :class:`uuid.UUID` instances will automatically be encoded by :mod:`bson` using this subtype. .. versionadded:: 2.1 """ UUID_SUBTYPE = 4 """BSON binary subtype for a UUID. This is the new BSON binary subtype for UUIDs. The current default is :data:`OLD_UUID_SUBTYPE` but will change to this in a future release. .. versionchanged:: 2.1 Changed to subtype 4. """ STANDARD = UUID_SUBTYPE """The standard UUID representation. :class:`uuid.UUID` instances will automatically be encoded to and decoded from BSON binary, using RFC-4122 byte order with binary subtype :data:`UUID_SUBTYPE`. .. versionadded:: 3.0 """ PYTHON_LEGACY = OLD_UUID_SUBTYPE """The Python legacy UUID representation. :class:`uuid.UUID` instances will automatically be encoded to and decoded from BSON binary, using RFC-4122 byte order with binary subtype :data:`OLD_UUID_SUBTYPE`. .. versionadded:: 3.0 """ JAVA_LEGACY = 5 """The Java legacy UUID representation. :class:`uuid.UUID` instances will automatically be encoded to and decoded from BSON binary subtype :data:`OLD_UUID_SUBTYPE`, using the Java driver's legacy byte order. .. versionchanged:: 3.6 BSON binary subtype 4 is decoded using RFC-4122 byte order. .. versionadded:: 2.3 """ CSHARP_LEGACY = 6 """The C#/.net legacy UUID representation. :class:`uuid.UUID` instances will automatically be encoded to and decoded from BSON binary subtype :data:`OLD_UUID_SUBTYPE`, using the C# driver's legacy byte order. .. versionchanged:: 3.6 BSON binary subtype 4 is decoded using RFC-4122 byte order. .. versionadded:: 2.3 """ ALL_UUID_SUBTYPES = (OLD_UUID_SUBTYPE, UUID_SUBTYPE) ALL_UUID_REPRESENTATIONS = (STANDARD, PYTHON_LEGACY, JAVA_LEGACY, CSHARP_LEGACY) UUID_REPRESENTATION_NAMES = { PYTHON_LEGACY: 'PYTHON_LEGACY', STANDARD: 'STANDARD', JAVA_LEGACY: 'JAVA_LEGACY', CSHARP_LEGACY: 'CSHARP_LEGACY'} MD5_SUBTYPE = 5 """BSON binary subtype for an MD5 hash. """ USER_DEFINED_SUBTYPE = 128 """BSON binary subtype for any user defined structure. """ class Binary(bytes): """Representation of BSON binary data. This is necessary because we want to represent Python strings as the BSON string type. We need to wrap binary data so we can tell the difference between what should be considered binary data and what should be considered a string when we encode to BSON. Raises TypeError if `data` is not an instance of :class:`str` (:class:`bytes` in python 3) or `subtype` is not an instance of :class:`int`. Raises ValueError if `subtype` is not in [0, 256). .. note:: In python 3 instances of Binary with subtype 0 will be decoded directly to :class:`bytes`. :Parameters: - `data`: the binary data to represent - `subtype` (optional): the `binary subtype `_ to use """ _type_marker = 5 def __new__(cls, data, subtype=BINARY_SUBTYPE): if not isinstance(data, bytes): raise TypeError("data must be an instance of bytes") if not isinstance(subtype, int): raise TypeError("subtype must be an instance of int") if subtype >= 256 or subtype < 0: raise ValueError("subtype must be contained in [0, 256)") self = bytes.__new__(cls, data) self.__subtype = subtype return self @property def subtype(self): """Subtype of this binary data. """ return self.__subtype def __getnewargs__(self): # Work around http://bugs.python.org/issue7382 data = super(Binary, self).__getnewargs__()[0] if PY3 and not isinstance(data, bytes): data = data.encode('latin-1') return data, self.__subtype def __eq__(self, other): if isinstance(other, Binary): return ((self.__subtype, bytes(self)) == (other.subtype, bytes(other))) # We don't return NotImplemented here because if we did then # Binary("foo") == "foo" would return True, since Binary is a # subclass of str... return False def __hash__(self): return super(Binary, self).__hash__() ^ hash(self.__subtype) def __ne__(self, other): return not self == other def __repr__(self): return "Binary(%s, %s)" % (bytes.__repr__(self), self.__subtype) class UUIDLegacy(Binary): """UUID wrapper to support working with UUIDs stored as PYTHON_LEGACY. .. doctest:: >>> import uuid >>> from bson.binary import Binary, UUIDLegacy, STANDARD >>> from bson.codec_options import CodecOptions >>> my_uuid = uuid.uuid4() >>> coll = db.get_collection('test', ... CodecOptions(uuid_representation=STANDARD)) >>> coll.insert_one({'uuid': Binary(my_uuid.bytes, 3)}).inserted_id ObjectId('...') >>> coll.find({'uuid': my_uuid}).count() 0 >>> coll.find({'uuid': UUIDLegacy(my_uuid)}).count() 1 >>> coll.find({'uuid': UUIDLegacy(my_uuid)})[0]['uuid'] UUID('...') >>> >>> # Convert from subtype 3 to subtype 4 >>> doc = coll.find_one({'uuid': UUIDLegacy(my_uuid)}) >>> coll.replace_one({"_id": doc["_id"]}, doc).matched_count 1 >>> coll.find({'uuid': UUIDLegacy(my_uuid)}).count() 0 >>> coll.find({'uuid': {'$in': [UUIDLegacy(my_uuid), my_uuid]}}).count() 1 >>> coll.find_one({'uuid': my_uuid})['uuid'] UUID('...') Raises TypeError if `obj` is not an instance of :class:`~uuid.UUID`. :Parameters: - `obj`: An instance of :class:`~uuid.UUID`. """ def __new__(cls, obj): if not isinstance(obj, UUID): raise TypeError("obj must be an instance of uuid.UUID") self = Binary.__new__(cls, obj.bytes, OLD_UUID_SUBTYPE) self.__uuid = obj return self def __getnewargs__(self): # Support copy and deepcopy return (self.__uuid,) @property def uuid(self): """UUID instance wrapped by this UUIDLegacy instance. """ return self.__uuid def __repr__(self): return "UUIDLegacy('%s')" % self.__uuid pymongo-3.6.1/bson/buffer.c0000644000076600000240000000770613156613521016026 0ustar shanestaff00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "buffer.h" #define INITIAL_BUFFER_SIZE 256 struct buffer { char* buffer; int size; int position; }; /* Allocate and return a new buffer. * Return NULL on allocation failure. */ buffer_t buffer_new(void) { buffer_t buffer; buffer = (buffer_t)malloc(sizeof(struct buffer)); if (buffer == NULL) { return NULL; } buffer->size = INITIAL_BUFFER_SIZE; buffer->position = 0; buffer->buffer = (char*)malloc(sizeof(char) * INITIAL_BUFFER_SIZE); if (buffer->buffer == NULL) { free(buffer); return NULL; } return buffer; } /* Free the memory allocated for `buffer`. * Return non-zero on failure. */ int buffer_free(buffer_t buffer) { if (buffer == NULL) { return 1; } free(buffer->buffer); free(buffer); return 0; } /* Grow `buffer` to at least `min_length`. * Return non-zero on allocation failure. */ static int buffer_grow(buffer_t buffer, int min_length) { int old_size = 0; int size = buffer->size; char* old_buffer = buffer->buffer; if (size >= min_length) { return 0; } while (size < min_length) { old_size = size; size *= 2; if (size <= old_size) { /* Size did not increase. Could be an overflow * or size < 1. Just go with min_length. */ size = min_length; } } buffer->buffer = (char*)realloc(buffer->buffer, sizeof(char) * size); if (buffer->buffer == NULL) { free(old_buffer); free(buffer); return 1; } buffer->size = size; return 0; } /* Assure that `buffer` has at least `size` free bytes (and grow if needed). * Return non-zero on allocation failure. */ static int buffer_assure_space(buffer_t buffer, int size) { if (buffer->position + size <= buffer->size) { return 0; } return buffer_grow(buffer, buffer->position + size); } /* Save `size` bytes from the current position in `buffer` (and grow if needed). * Return offset for writing, or -1 on allocation failure. */ buffer_position buffer_save_space(buffer_t buffer, int size) { int position = buffer->position; if (buffer_assure_space(buffer, size) != 0) { return -1; } buffer->position += size; return position; } /* Write `size` bytes from `data` to `buffer` (and grow if needed). * Return non-zero on allocation failure. */ int buffer_write(buffer_t buffer, const char* data, int size) { if (buffer_assure_space(buffer, size) != 0) { return 1; } memcpy(buffer->buffer + buffer->position, data, size); buffer->position += size; return 0; } /* Write `size` bytes from `data` to `buffer` at position `position`. * Does not change the internal position of `buffer`. * Return non-zero if buffer isn't large enough for write. */ int buffer_write_at_position(buffer_t buffer, buffer_position position, const char* data, int size) { if (position + size > buffer->size) { buffer_free(buffer); return 1; } memcpy(buffer->buffer + position, data, size); return 0; } int buffer_get_position(buffer_t buffer) { return buffer->position; } char* buffer_get_buffer(buffer_t buffer) { return buffer->buffer; } void buffer_update_position(buffer_t buffer, buffer_position new_position) { buffer->position = new_position; } pymongo-3.6.1/bson/encoding_helpers.c0000644000076600000240000001060013156613521020050 0ustar shanestaff00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "encoding_helpers.h" /* * Portions Copyright 2001 Unicode, Inc. * * Disclaimer * * This source code is provided as is by Unicode, Inc. No claims are * made as to fitness for any particular purpose. No warranties of any * kind are expressed or implied. The recipient agrees to determine * applicability of information provided. If this file has been * purchased on magnetic or optical media from Unicode, Inc., the * sole remedy for any claim will be exchange of defective media * within 90 days of receipt. * * Limitations on Rights to Redistribute This Code * * Unicode, Inc. hereby grants the right to freely use the information * supplied in this file in the creation of products supporting the * Unicode Standard, and to make copies of this file in any form * for internal or external distribution as long as this notice * remains attached. */ /* * Index into the table below with the first byte of a UTF-8 sequence to * get the number of trailing bytes that are supposed to follow it. */ static const char trailingBytesForUTF8[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 }; /* --------------------------------------------------------------------- */ /* * Utility routine to tell whether a sequence of bytes is legal UTF-8. * This must be called with the length pre-determined by the first byte. * The length can be set by: * length = trailingBytesForUTF8[*source]+1; * and the sequence is illegal right away if there aren't that many bytes * available. * If presented with a length > 4, this returns 0. The Unicode * definition of UTF-8 goes up to 4-byte sequences. */ static unsigned char isLegalUTF8(const unsigned char* source, int length) { unsigned char a; const unsigned char* srcptr = source + length; switch (length) { default: return 0; /* Everything else falls through when "true"... */ case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0; case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0; case 2: if ((a = (*--srcptr)) > 0xBF) return 0; switch (*source) { /* no fall-through in this inner switch */ case 0xE0: if (a < 0xA0) return 0; break; case 0xF0: if (a < 0x90) return 0; break; case 0xF4: if (a > 0x8F) return 0; break; default: if (a < 0x80) return 0; } case 1: if (*source >= 0x80 && *source < 0xC2) return 0; if (*source > 0xF4) return 0; } return 1; } result_t check_string(const unsigned char* string, const int length, const char check_utf8, const char check_null) { int position = 0; /* By default we go character by character. Will be different for checking * UTF-8 */ int sequence_length = 1; if (!check_utf8 && !check_null) { return VALID; } while (position < length) { if (check_null && *(string + position) == 0) { return HAS_NULL; } if (check_utf8) { sequence_length = trailingBytesForUTF8[*(string + position)] + 1; if ((position + sequence_length) > length) { return NOT_UTF_8; } if (!isLegalUTF8(string + position, sequence_length)) { return NOT_UTF_8; } } position += sequence_length; } return VALID; } pymongo-3.6.1/bson/py3compat.py0000644000076600000240000000462713245621354016703 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Utility functions and definitions for python3 compatibility.""" import sys PY3 = sys.version_info[0] == 3 if PY3: import codecs import _thread as thread from io import BytesIO as StringIO try: import collections.abc as abc except ImportError: # PyPy3 (based on CPython 3.2) import collections as abc MAXSIZE = sys.maxsize imap = map def b(s): # BSON and socket operations deal in binary data. In # python 3 that means instances of `bytes`. In python # 2.6 and 2.7 you can create an alias for `bytes` using # the b prefix (e.g. b'foo'). # See http://python3porting.com/problems.html#nicer-solutions return codecs.latin_1_encode(s)[0] def bytes_from_hex(h): return bytes.fromhex(h) def iteritems(d): return iter(d.items()) def itervalues(d): return iter(d.values()) def reraise(exctype, value, trace=None): raise exctype(str(value)).with_traceback(trace) def _unicode(s): return s text_type = str string_type = str integer_types = int else: import collections as abc import thread from itertools import imap try: from cStringIO import StringIO except ImportError: from StringIO import StringIO MAXSIZE = sys.maxint def b(s): # See comments above. In python 2.x b('foo') is just 'foo'. return s def bytes_from_hex(h): return h.decode('hex') def iteritems(d): return d.iteritems() def itervalues(d): return d.itervalues() # "raise x, y, z" raises SyntaxError in Python 3 exec("""def reraise(exctype, value, trace=None): raise exctype, str(value), trace """) _unicode = unicode string_type = basestring text_type = unicode integer_types = (int, long) pymongo-3.6.1/bson/raw_bson.py0000644000076600000240000000737713245621354016603 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing raw BSON documents. """ from bson import _UNPACK_INT, _iterate_elements from bson.py3compat import abc, iteritems from bson.codec_options import ( DEFAULT_CODEC_OPTIONS as DEFAULT, _RAW_BSON_DOCUMENT_MARKER) from bson.errors import InvalidBSON class RawBSONDocument(abc.Mapping): """Representation for a MongoDB document that provides access to the raw BSON bytes that compose it. Only when a field is accessed or modified within the document does RawBSONDocument decode its bytes. """ __slots__ = ('__raw', '__inflated_doc', '__codec_options') _type_marker = _RAW_BSON_DOCUMENT_MARKER def __init__(self, bson_bytes, codec_options=None): """Create a new :class:`RawBSONDocument`. :Parameters: - `bson_bytes`: the BSON bytes that compose this document - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. .. versionchanged:: 3.5 If a :class:`~bson.codec_options.CodecOptions` is passed in, its `document_class` must be :class:`RawBSONDocument`. """ self.__raw = bson_bytes self.__inflated_doc = None # Can't default codec_options to DEFAULT_RAW_BSON_OPTIONS in signature, # it refers to this class RawBSONDocument. if codec_options is None: codec_options = DEFAULT_RAW_BSON_OPTIONS elif codec_options.document_class is not RawBSONDocument: raise TypeError( "RawBSONDocument cannot use CodecOptions with document " "class %s" % (codec_options.document_class, )) self.__codec_options = codec_options @property def raw(self): """The raw BSON bytes composing this document.""" return self.__raw def items(self): """Lazily decode and iterate elements in this document.""" return iteritems(self.__inflated) @property def __inflated(self): if self.__inflated_doc is None: # We already validated the object's size when this document was # created, so no need to do that again. We still need to check the # size of all the elements and compare to the document size. object_size = _UNPACK_INT(self.__raw[:4])[0] - 1 position = 0 self.__inflated_doc = {} for key, value, position in _iterate_elements( self.__raw, 4, object_size, self.__codec_options): self.__inflated_doc[key] = value if position != object_size: raise InvalidBSON('bad object or element length') return self.__inflated_doc def __getitem__(self, item): return self.__inflated[item] def __iter__(self): return iter(self.__inflated) def __len__(self): return len(self.__inflated) def __eq__(self, other): if isinstance(other, RawBSONDocument): return self.__raw == other.raw return NotImplemented def __repr__(self): return ("RawBSONDocument(%r, codec_options=%r)" % (self.raw, self.__codec_options)) DEFAULT_RAW_BSON_OPTIONS = DEFAULT.with_options(document_class=RawBSONDocument) pymongo-3.6.1/bson/time64.c0000644000076600000240000005146113245621354015664 0ustar shanestaff00000000000000/* Copyright (c) 2007-2010 Michael G Schwern This software originally derived from Paul Sheer's pivotal_gmtime_r.c. The MIT License: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* Programmers who have available to them 64-bit time values as a 'long long' type can use localtime64_r() and gmtime64_r() which correctly converts the time even on 32-bit systems. Whether you have 64-bit time values will depend on the operating system. localtime64_r() is a 64-bit equivalent of localtime_r(). gmtime64_r() is a 64-bit equivalent of gmtime_r(). */ #ifdef _MSC_VER #define _CRT_SECURE_NO_WARNINGS #endif /* Including Python.h fixes issues with interpreters built with -std=c99. */ #include "Python.h" #include #include "time64.h" #include "time64_limits.h" /* Spec says except for stftime() and the _r() functions, these all return static memory. Stabbings! */ static struct TM Static_Return_Date; static const int days_in_month[2][12] = { {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}, {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}, }; static const int julian_days_by_month[2][12] = { {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}, {0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335}, }; static const int length_of_year[2] = { 365, 366 }; /* Some numbers relating to the gregorian cycle */ static const Year years_in_gregorian_cycle = 400; #define days_in_gregorian_cycle ((365 * 400) + 100 - 4 + 1) static const Time64_T seconds_in_gregorian_cycle = days_in_gregorian_cycle * 60LL * 60LL * 24LL; /* Year range we can trust the time funcitons with */ #define MAX_SAFE_YEAR 2037 #define MIN_SAFE_YEAR 1971 /* 28 year Julian calendar cycle */ #define SOLAR_CYCLE_LENGTH 28 /* Year cycle from MAX_SAFE_YEAR down. */ static const int safe_years_high[SOLAR_CYCLE_LENGTH] = { 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2010, 2011, 2012, 2013, 2014, 2015 }; /* Year cycle from MIN_SAFE_YEAR up */ static const int safe_years_low[SOLAR_CYCLE_LENGTH] = { 1996, 1997, 1998, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, }; /* Let's assume people are going to be looking for dates in the future. Let's provide some cheats so you can skip ahead. This has a 4x speed boost when near 2008. */ /* Number of days since epoch on Jan 1st, 2008 GMT */ #define CHEAT_DAYS (1199145600 / 24 / 60 / 60) #define CHEAT_YEARS 108 #define IS_LEAP(n) ((!(((n) + 1900) % 400) || (!(((n) + 1900) % 4) && (((n) + 1900) % 100))) != 0) #define _TIME64_WRAP(a,b,m) ((a) = ((a) < 0 ) ? ((b)--, (a) + (m)) : (a)) #ifdef USE_SYSTEM_LOCALTIME # define SHOULD_USE_SYSTEM_LOCALTIME(a) ( \ (a) <= SYSTEM_LOCALTIME_MAX && \ (a) >= SYSTEM_LOCALTIME_MIN \ ) #else # define SHOULD_USE_SYSTEM_LOCALTIME(a) (0) #endif #ifdef USE_SYSTEM_GMTIME # define SHOULD_USE_SYSTEM_GMTIME(a) ( \ (a) <= SYSTEM_GMTIME_MAX && \ (a) >= SYSTEM_GMTIME_MIN \ ) #else # define SHOULD_USE_SYSTEM_GMTIME(a) (0) #endif /* Multi varadic macros are a C99 thing, alas */ #ifdef TIME_64_DEBUG # define TIME64_TRACE(format) (fprintf(stderr, format)) # define TIME64_TRACE1(format, var1) (fprintf(stderr, format, var1)) # define TIME64_TRACE2(format, var1, var2) (fprintf(stderr, format, var1, var2)) # define TIME64_TRACE3(format, var1, var2, var3) (fprintf(stderr, format, var1, var2, var3)) #else # define TIME64_TRACE(format) ((void)0) # define TIME64_TRACE1(format, var1) ((void)0) # define TIME64_TRACE2(format, var1, var2) ((void)0) # define TIME64_TRACE3(format, var1, var2, var3) ((void)0) #endif static int is_exception_century(Year year) { int is_exception = ((year % 100 == 0) && !(year % 400 == 0)); TIME64_TRACE1("# is_exception_century: %s\n", is_exception ? "yes" : "no"); return(is_exception); } /* Compare two dates. The result is like cmp. Ignores things like gmtoffset and dst */ int cmp_date( const struct TM* left, const struct tm* right ) { if( left->tm_year > right->tm_year ) return 1; else if( left->tm_year < right->tm_year ) return -1; if( left->tm_mon > right->tm_mon ) return 1; else if( left->tm_mon < right->tm_mon ) return -1; if( left->tm_mday > right->tm_mday ) return 1; else if( left->tm_mday < right->tm_mday ) return -1; if( left->tm_hour > right->tm_hour ) return 1; else if( left->tm_hour < right->tm_hour ) return -1; if( left->tm_min > right->tm_min ) return 1; else if( left->tm_min < right->tm_min ) return -1; if( left->tm_sec > right->tm_sec ) return 1; else if( left->tm_sec < right->tm_sec ) return -1; return 0; } /* Check if a date is safely inside a range. The intention is to check if its a few days inside. */ int date_in_safe_range( const struct TM* date, const struct tm* min, const struct tm* max ) { if( cmp_date(date, min) == -1 ) return 0; if( cmp_date(date, max) == 1 ) return 0; return 1; } /* timegm() is not in the C or POSIX spec, but it is such a useful extension I would be remiss in leaving it out. Also I need it for localtime64() */ Time64_T timegm64(const struct TM *date) { Time64_T days = 0; Time64_T seconds = 0; Year year; Year orig_year = (Year)date->tm_year; int cycles = 0; if( orig_year > 100 ) { cycles = (int)((orig_year - 100) / 400); orig_year -= cycles * 400; days += (Time64_T)cycles * days_in_gregorian_cycle; } else if( orig_year < -300 ) { cycles = (int)((orig_year - 100) / 400); orig_year -= cycles * 400; days += (Time64_T)cycles * days_in_gregorian_cycle; } TIME64_TRACE3("# timegm/ cycles: %d, days: %lld, orig_year: %lld\n", cycles, days, orig_year); if( orig_year > 70 ) { year = 70; while( year < orig_year ) { days += length_of_year[IS_LEAP(year)]; year++; } } else if ( orig_year < 70 ) { year = 69; do { days -= length_of_year[IS_LEAP(year)]; year--; } while( year >= orig_year ); } days += julian_days_by_month[IS_LEAP(orig_year)][date->tm_mon]; days += date->tm_mday - 1; seconds = days * 60 * 60 * 24; seconds += date->tm_hour * 60 * 60; seconds += date->tm_min * 60; seconds += date->tm_sec; return(seconds); } #ifndef NDEBUG static int check_tm(struct TM *tm) { /* Don't forget leap seconds */ assert(tm->tm_sec >= 0); assert(tm->tm_sec <= 61); assert(tm->tm_min >= 0); assert(tm->tm_min <= 59); assert(tm->tm_hour >= 0); assert(tm->tm_hour <= 23); assert(tm->tm_mday >= 1); assert(tm->tm_mday <= days_in_month[IS_LEAP(tm->tm_year)][tm->tm_mon]); assert(tm->tm_mon >= 0); assert(tm->tm_mon <= 11); assert(tm->tm_wday >= 0); assert(tm->tm_wday <= 6); assert(tm->tm_yday >= 0); assert(tm->tm_yday <= length_of_year[IS_LEAP(tm->tm_year)]); #ifdef HAS_TM_TM_GMTOFF assert(tm->tm_gmtoff >= -24 * 60 * 60); assert(tm->tm_gmtoff <= 24 * 60 * 60); #endif return 1; } #endif /* The exceptional centuries without leap years cause the cycle to shift by 16 */ static Year cycle_offset(Year year) { const Year start_year = 2000; Year year_diff = year - start_year; Year exceptions; if( year > start_year ) year_diff--; exceptions = year_diff / 100; exceptions -= year_diff / 400; TIME64_TRACE3("# year: %lld, exceptions: %lld, year_diff: %lld\n", year, exceptions, year_diff); return exceptions * 16; } /* For a given year after 2038, pick the latest possible matching year in the 28 year calendar cycle. A matching year... 1) Starts on the same day of the week. 2) Has the same leap year status. This is so the calendars match up. Also the previous year must match. When doing Jan 1st you might wind up on Dec 31st the previous year when doing a -UTC time zone. Finally, the next year must have the same start day of week. This is for Dec 31st with a +UTC time zone. It doesn't need the same leap year status since we only care about January 1st. */ static int safe_year(const Year year) { int safe_year = 0; Year year_cycle; if( year >= MIN_SAFE_YEAR && year <= MAX_SAFE_YEAR ) { return (int)year; } year_cycle = year + cycle_offset(year); /* safe_years_low is off from safe_years_high by 8 years */ if( year < MIN_SAFE_YEAR ) year_cycle -= 8; /* Change non-leap xx00 years to an equivalent */ if( is_exception_century(year) ) year_cycle += 11; /* Also xx01 years, since the previous year will be wrong */ if( is_exception_century(year - 1) ) year_cycle += 17; year_cycle %= SOLAR_CYCLE_LENGTH; if( year_cycle < 0 ) year_cycle = SOLAR_CYCLE_LENGTH + year_cycle; assert( year_cycle >= 0 ); assert( year_cycle < SOLAR_CYCLE_LENGTH ); if( year < MIN_SAFE_YEAR ) safe_year = safe_years_low[year_cycle]; else if( year > MAX_SAFE_YEAR ) safe_year = safe_years_high[year_cycle]; else assert(0); TIME64_TRACE3("# year: %lld, year_cycle: %lld, safe_year: %d\n", year, year_cycle, safe_year); assert(safe_year <= MAX_SAFE_YEAR && safe_year >= MIN_SAFE_YEAR); return safe_year; } void copy_tm_to_TM64(const struct tm *src, struct TM *dest) { if( src == NULL ) { memset(dest, 0, sizeof(*dest)); } else { # ifdef USE_TM64 dest->tm_sec = src->tm_sec; dest->tm_min = src->tm_min; dest->tm_hour = src->tm_hour; dest->tm_mday = src->tm_mday; dest->tm_mon = src->tm_mon; dest->tm_year = (Year)src->tm_year; dest->tm_wday = src->tm_wday; dest->tm_yday = src->tm_yday; dest->tm_isdst = src->tm_isdst; # ifdef HAS_TM_TM_GMTOFF dest->tm_gmtoff = src->tm_gmtoff; # endif # ifdef HAS_TM_TM_ZONE dest->tm_zone = src->tm_zone; # endif # else /* They're the same type */ memcpy(dest, src, sizeof(*dest)); # endif } } void copy_TM64_to_tm(const struct TM *src, struct tm *dest) { if( src == NULL ) { memset(dest, 0, sizeof(*dest)); } else { # ifdef USE_TM64 dest->tm_sec = src->tm_sec; dest->tm_min = src->tm_min; dest->tm_hour = src->tm_hour; dest->tm_mday = src->tm_mday; dest->tm_mon = src->tm_mon; dest->tm_year = (int)src->tm_year; dest->tm_wday = src->tm_wday; dest->tm_yday = src->tm_yday; dest->tm_isdst = src->tm_isdst; # ifdef HAS_TM_TM_GMTOFF dest->tm_gmtoff = src->tm_gmtoff; # endif # ifdef HAS_TM_TM_ZONE dest->tm_zone = src->tm_zone; # endif # else /* They're the same type */ memcpy(dest, src, sizeof(*dest)); # endif } } /* Simulate localtime_r() to the best of our ability */ struct tm * fake_localtime_r(const time_t *time, struct tm *result) { const struct tm *static_result = localtime(time); assert(result != NULL); if( static_result == NULL ) { memset(result, 0, sizeof(*result)); return NULL; } else { memcpy(result, static_result, sizeof(*result)); return result; } } /* Simulate gmtime_r() to the best of our ability */ struct tm * fake_gmtime_r(const time_t *time, struct tm *result) { const struct tm *static_result = gmtime(time); assert(result != NULL); if( static_result == NULL ) { memset(result, 0, sizeof(*result)); return NULL; } else { memcpy(result, static_result, sizeof(*result)); return result; } } static Time64_T seconds_between_years(Year left_year, Year right_year) { int increment = (left_year > right_year) ? 1 : -1; Time64_T seconds = 0; int cycles; if( left_year > 2400 ) { cycles = (int)((left_year - 2400) / 400); left_year -= cycles * 400; seconds += cycles * seconds_in_gregorian_cycle; } else if( left_year < 1600 ) { cycles = (int)((left_year - 1600) / 400); left_year += cycles * 400; seconds += cycles * seconds_in_gregorian_cycle; } while( left_year != right_year ) { seconds += length_of_year[IS_LEAP(right_year - 1900)] * 60 * 60 * 24; right_year += increment; } return seconds * increment; } Time64_T mktime64(const struct TM *input_date) { struct tm safe_date; struct TM date; Time64_T time; Year year = input_date->tm_year + 1900; if( date_in_safe_range(input_date, &SYSTEM_MKTIME_MIN, &SYSTEM_MKTIME_MAX) ) { copy_TM64_to_tm(input_date, &safe_date); return (Time64_T)mktime(&safe_date); } /* Have to make the year safe in date else it won't fit in safe_date */ date = *input_date; date.tm_year = safe_year(year) - 1900; copy_TM64_to_tm(&date, &safe_date); time = (Time64_T)mktime(&safe_date); time += seconds_between_years(year, (Year)(safe_date.tm_year + 1900)); return time; } /* Because I think mktime() is a crappy name */ Time64_T timelocal64(const struct TM *date) { return mktime64(date); } struct TM *gmtime64_r (const Time64_T *in_time, struct TM *p) { int v_tm_sec, v_tm_min, v_tm_hour, v_tm_mon, v_tm_wday; Time64_T v_tm_tday; int leap; Time64_T m; Time64_T time = *in_time; Year year = 70; int cycles = 0; assert(p != NULL); #ifdef USE_SYSTEM_GMTIME /* Use the system gmtime() if time_t is small enough */ if( SHOULD_USE_SYSTEM_GMTIME(*in_time) ) { time_t safe_time = (time_t)*in_time; struct tm safe_date; GMTIME_R(&safe_time, &safe_date); copy_tm_to_TM64(&safe_date, p); assert(check_tm(p)); return p; } #endif #ifdef HAS_TM_TM_GMTOFF p->tm_gmtoff = 0; #endif p->tm_isdst = 0; #ifdef HAS_TM_TM_ZONE p->tm_zone = "UTC"; #endif v_tm_sec = (int)(time % 60); time /= 60; v_tm_min = (int)(time % 60); time /= 60; v_tm_hour = (int)(time % 24); time /= 24; v_tm_tday = time; _TIME64_WRAP (v_tm_sec, v_tm_min, 60); _TIME64_WRAP (v_tm_min, v_tm_hour, 60); _TIME64_WRAP (v_tm_hour, v_tm_tday, 24); v_tm_wday = (int)((v_tm_tday + 4) % 7); if (v_tm_wday < 0) v_tm_wday += 7; m = v_tm_tday; if (m >= CHEAT_DAYS) { year = CHEAT_YEARS; m -= CHEAT_DAYS; } if (m >= 0) { /* Gregorian cycles, this is huge optimization for distant times */ cycles = (int)(m / (Time64_T) days_in_gregorian_cycle); if( cycles ) { m -= (cycles * (Time64_T) days_in_gregorian_cycle); year += (cycles * years_in_gregorian_cycle); } /* Years */ leap = IS_LEAP (year); while (m >= (Time64_T) length_of_year[leap]) { m -= (Time64_T) length_of_year[leap]; year++; leap = IS_LEAP (year); } /* Months */ v_tm_mon = 0; while (m >= (Time64_T) days_in_month[leap][v_tm_mon]) { m -= (Time64_T) days_in_month[leap][v_tm_mon]; v_tm_mon++; } } else { year--; /* Gregorian cycles */ cycles = (int)((m / (Time64_T) days_in_gregorian_cycle) + 1); if( cycles ) { m -= (cycles * (Time64_T) days_in_gregorian_cycle); year += (cycles * years_in_gregorian_cycle); } /* Years */ leap = IS_LEAP (year); while (m < (Time64_T) -length_of_year[leap]) { m += (Time64_T) length_of_year[leap]; year--; leap = IS_LEAP (year); } /* Months */ v_tm_mon = 11; while (m < (Time64_T) -days_in_month[leap][v_tm_mon]) { m += (Time64_T) days_in_month[leap][v_tm_mon]; v_tm_mon--; } m += (Time64_T) days_in_month[leap][v_tm_mon]; } p->tm_year = (int)year; if( p->tm_year != year ) { #ifdef EOVERFLOW errno = EOVERFLOW; #endif return NULL; } /* At this point m is less than a year so casting to an int is safe */ p->tm_mday = (int) m + 1; p->tm_yday = julian_days_by_month[leap][v_tm_mon] + (int)m; p->tm_sec = v_tm_sec; p->tm_min = v_tm_min; p->tm_hour = v_tm_hour; p->tm_mon = v_tm_mon; p->tm_wday = v_tm_wday; assert(check_tm(p)); return p; } struct TM *localtime64_r (const Time64_T *time, struct TM *local_tm) { time_t safe_time; struct tm safe_date; struct TM gm_tm; Year orig_year; int month_diff; assert(local_tm != NULL); #ifdef USE_SYSTEM_LOCALTIME /* Use the system localtime() if time_t is small enough */ if( SHOULD_USE_SYSTEM_LOCALTIME(*time) ) { safe_time = (time_t)*time; TIME64_TRACE1("Using system localtime for %lld\n", *time); LOCALTIME_R(&safe_time, &safe_date); copy_tm_to_TM64(&safe_date, local_tm); assert(check_tm(local_tm)); return local_tm; } #endif if( gmtime64_r(time, &gm_tm) == NULL ) { TIME64_TRACE1("gmtime64_r returned null for %lld\n", *time); return NULL; } orig_year = gm_tm.tm_year; if (gm_tm.tm_year > (2037 - 1900) || gm_tm.tm_year < (1970 - 1900) ) { TIME64_TRACE1("Mapping tm_year %lld to safe_year\n", (Year)gm_tm.tm_year); gm_tm.tm_year = safe_year((Year)(gm_tm.tm_year + 1900)) - 1900; } safe_time = (time_t)timegm64(&gm_tm); if( LOCALTIME_R(&safe_time, &safe_date) == NULL ) { TIME64_TRACE1("localtime_r(%d) returned NULL\n", (int)safe_time); return NULL; } copy_tm_to_TM64(&safe_date, local_tm); local_tm->tm_year = (int)orig_year; if( local_tm->tm_year != orig_year ) { TIME64_TRACE2("tm_year overflow: tm_year %lld, orig_year %lld\n", (Year)local_tm->tm_year, (Year)orig_year); #ifdef EOVERFLOW errno = EOVERFLOW; #endif return NULL; } month_diff = local_tm->tm_mon - gm_tm.tm_mon; /* When localtime is Dec 31st previous year and gmtime is Jan 1st next year. */ if( month_diff == 11 ) { local_tm->tm_year--; } /* When localtime is Jan 1st, next year and gmtime is Dec 31st, previous year. */ if( month_diff == -11 ) { local_tm->tm_year++; } /* GMT is Jan 1st, xx01 year, but localtime is still Dec 31st in a non-leap xx00. There is one point in the cycle we can't account for which the safe xx00 year is a leap year. So we need to correct for Dec 31st comming out as the 366th day of the year. */ if( !IS_LEAP(local_tm->tm_year) && local_tm->tm_yday == 365 ) local_tm->tm_yday--; assert(check_tm(local_tm)); return local_tm; } int valid_tm_wday( const struct TM* date ) { if( 0 <= date->tm_wday && date->tm_wday <= 6 ) return 1; else return 0; } int valid_tm_mon( const struct TM* date ) { if( 0 <= date->tm_mon && date->tm_mon <= 11 ) return 1; else return 0; } /* Non-thread safe versions of the above */ struct TM *localtime64(const Time64_T *time) { #ifdef _MSC_VER _tzset(); #else tzset(); #endif return localtime64_r(time, &Static_Return_Date); } struct TM *gmtime64(const Time64_T *time) { return gmtime64_r(time, &Static_Return_Date); } pymongo-3.6.1/bson/decimal128.py0000644000076600000240000002537213245621354016615 0ustar shanestaff00000000000000# Copyright 2016-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for working with the BSON decimal128 type. .. versionadded:: 3.4 .. note:: The Decimal128 BSON type requires MongoDB 3.4+. """ import decimal import struct import sys from bson.py3compat import (PY3 as _PY3, string_type as _string_type) if _PY3: _from_bytes = int.from_bytes # pylint: disable=no-member, invalid-name else: import binascii def _from_bytes(value, dummy, _int=int, _hexlify=binascii.hexlify): "An implementation of int.from_bytes for python 2.x." return _int(_hexlify(value), 16) if sys.version_info[:2] == (2, 6): def _bit_length(num): """bit_length for python 2.6""" if num: # bin() was new in 2.6. Note that this won't work # for values less than 0, which we never have here. return len(bin(num)) - 2 # bit_length(0) is 0, but len(bin(0)) - 2 is 1 return 0 else: def _bit_length(num): """bit_length for python >= 2.7""" # num could be int or long in python 2.7 return num.bit_length() _PACK_64 = struct.Struct("= 3.3, cdecimal decimal.Context(clamp=1) # pylint: disable=unexpected-keyword-arg _CTX_OPTIONS['clamp'] = 1 except TypeError: # Python < 3.3 _CTX_OPTIONS['_clamp'] = 1 _DEC128_CTX = decimal.Context(**_CTX_OPTIONS.copy()) def create_decimal128_context(): """Returns an instance of :class:`decimal.Context` appropriate for working with IEEE-754 128-bit decimal floating point values. """ opts = _CTX_OPTIONS.copy() opts['traps'] = [] return decimal.Context(**opts) def _decimal_to_128(value): """Converts a decimal.Decimal to BID (high bits, low bits). :Parameters: - `value`: An instance of decimal.Decimal """ with decimal.localcontext(_DEC128_CTX) as ctx: value = ctx.create_decimal(value) if value.is_infinite(): return _NINF if value.is_signed() else _PINF sign, digits, exponent = value.as_tuple() if value.is_nan(): if digits: raise ValueError("NaN with debug payload is not supported") if value.is_snan(): return _NSNAN if value.is_signed() else _PSNAN return _NNAN if value.is_signed() else _PNAN significand = int("".join([str(digit) for digit in digits])) bit_length = _bit_length(significand) high = 0 low = 0 for i in range(min(64, bit_length)): if significand & (1 << i): low |= 1 << i for i in range(64, bit_length): if significand & (1 << i): high |= 1 << (i - 64) biased_exponent = exponent + _EXPONENT_BIAS if high >> 49 == 1: high = high & 0x7fffffffffff high |= _EXPONENT_MASK high |= (biased_exponent & 0x3fff) << 47 else: high |= biased_exponent << 49 if sign: high |= _SIGN return high, low class Decimal128(object): """BSON Decimal128 type:: >>> Decimal128(Decimal("0.0005")) Decimal128('0.0005') >>> Decimal128("0.0005") Decimal128('0.0005') >>> Decimal128((3474527112516337664, 5)) Decimal128('0.0005') :Parameters: - `value`: An instance of :class:`decimal.Decimal`, string, or tuple of (high bits, low bits) from Binary Integer Decimal (BID) format. .. note:: :class:`~Decimal128` uses an instance of :class:`decimal.Context` configured for IEEE-754 Decimal128 when validating parameters. Signals like :class:`decimal.InvalidOperation`, :class:`decimal.Inexact`, and :class:`decimal.Overflow` are trapped and raised as exceptions:: >>> Decimal128(".13.1") Traceback (most recent call last): File "", line 1, in ... decimal.InvalidOperation: [] >>> >>> Decimal128("1E-6177") Traceback (most recent call last): File "", line 1, in ... decimal.Inexact: [] >>> >>> Decimal128("1E6145") Traceback (most recent call last): File "", line 1, in ... decimal.Overflow: [, ] To ensure the result of a calculation can always be stored as BSON Decimal128 use the context returned by :func:`create_decimal128_context`:: >>> import decimal >>> decimal128_ctx = create_decimal128_context() >>> with decimal.localcontext(decimal128_ctx) as ctx: ... Decimal128(ctx.create_decimal(".13.3")) ... Decimal128('NaN') >>> >>> with decimal.localcontext(decimal128_ctx) as ctx: ... Decimal128(ctx.create_decimal("1E-6177")) ... Decimal128('0E-6176') >>> >>> with decimal.localcontext(DECIMAL128_CTX) as ctx: ... Decimal128(ctx.create_decimal("1E6145")) ... Decimal128('Infinity') To match the behavior of MongoDB's Decimal128 implementation str(Decimal(value)) may not match str(Decimal128(value)) for NaN values:: >>> Decimal128(Decimal('NaN')) Decimal128('NaN') >>> Decimal128(Decimal('-NaN')) Decimal128('NaN') >>> Decimal128(Decimal('sNaN')) Decimal128('NaN') >>> Decimal128(Decimal('-sNaN')) Decimal128('NaN') However, :meth:`~Decimal128.to_decimal` will return the exact value:: >>> Decimal128(Decimal('NaN')).to_decimal() Decimal('NaN') >>> Decimal128(Decimal('-NaN')).to_decimal() Decimal('-NaN') >>> Decimal128(Decimal('sNaN')).to_decimal() Decimal('sNaN') >>> Decimal128(Decimal('-sNaN')).to_decimal() Decimal('-sNaN') Two instances of :class:`Decimal128` compare equal if their Binary Integer Decimal encodings are equal:: >>> Decimal128('NaN') == Decimal128('NaN') True >>> Decimal128('NaN').bid == Decimal128('NaN').bid True This differs from :class:`decimal.Decimal` comparisons for NaN:: >>> Decimal('NaN') == Decimal('NaN') False """ __slots__ = ('__high', '__low') _type_marker = 19 def __init__(self, value): if isinstance(value, (_string_type, decimal.Decimal)): self.__high, self.__low = _decimal_to_128(value) elif isinstance(value, (list, tuple)): if len(value) != 2: raise ValueError('Invalid size for creation of Decimal128 ' 'from list or tuple. Must have exactly 2 ' 'elements.') self.__high, self.__low = value else: raise TypeError("Cannot convert %r to Decimal128" % (value,)) def to_decimal(self): """Returns an instance of :class:`decimal.Decimal` for this :class:`Decimal128`. """ high = self.__high low = self.__low sign = 1 if (high & _SIGN) else 0 if (high & _SNAN) == _SNAN: return decimal.Decimal((sign, (), 'N')) elif (high & _NAN) == _NAN: return decimal.Decimal((sign, (), 'n')) elif (high & _INF) == _INF: return decimal.Decimal((sign, (), 'F')) if (high & _EXPONENT_MASK) == _EXPONENT_MASK: exponent = ((high & 0x1fffe00000000000) >> 47) - _EXPONENT_BIAS return decimal.Decimal((sign, (0,), exponent)) else: exponent = ((high & 0x7fff800000000000) >> 49) - _EXPONENT_BIAS arr = bytearray(15) mask = 0x00000000000000ff for i in range(14, 6, -1): arr[i] = (low & mask) >> ((14 - i) << 3) mask = mask << 8 mask = 0x00000000000000ff for i in range(6, 0, -1): arr[i] = (high & mask) >> ((6 - i) << 3) mask = mask << 8 mask = 0x0001000000000000 arr[0] = (high & mask) >> 48 # Have to convert bytearray to bytes for python 2.6. # cdecimal only accepts a tuple for digits. digits = tuple( int(digit) for digit in str(_from_bytes(bytes(arr), 'big'))) with decimal.localcontext(_DEC128_CTX) as ctx: return ctx.create_decimal((sign, digits, exponent)) @classmethod def from_bid(cls, value): """Create an instance of :class:`Decimal128` from Binary Integer Decimal string. :Parameters: - `value`: 16 byte string (128-bit IEEE 754-2008 decimal floating point in Binary Integer Decimal (BID) format). """ if not isinstance(value, bytes): raise TypeError("value must be an instance of bytes") if len(value) != 16: raise ValueError("value must be exactly 16 bytes") return cls((_UNPACK_64(value[8:])[0], _UNPACK_64(value[:8])[0])) @property def bid(self): """The Binary Integer Decimal (BID) encoding of this instance.""" return _PACK_64(self.__low) + _PACK_64(self.__high) def __str__(self): dec = self.to_decimal() if dec.is_nan(): # Required by the drivers spec to match MongoDB behavior. return "NaN" return str(dec) def __repr__(self): return "Decimal128('%s')" % (str(self),) def __setstate__(self, value): self.__high, self.__low = value def __getstate__(self): return self.__high, self.__low def __eq__(self, other): if isinstance(other, Decimal128): return self.bid == other.bid return NotImplemented def __ne__(self, other): return not self == other pymongo-3.6.1/bson/_cbsonmodule.h0000644000076600000240000001615713245621354017235 0ustar shanestaff00000000000000/* * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "bson-endian.h" #ifndef _CBSONMODULE_H #define _CBSONMODULE_H /* Py_ssize_t was new in python 2.5. See conversion * guidlines in http://www.python.org/dev/peps/pep-0353 * */ #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) typedef int Py_ssize_t; #define PY_SSIZE_T_MAX INT_MAX #define PY_SSIZE_T_MIN INT_MIN #endif #if defined(WIN32) || defined(_MSC_VER) /* * This macro is basically an implementation of asprintf for win32 * We print to the provided buffer to get the string value as an int. */ #if defined(_MSC_VER) && (_MSC_VER >= 1400) #define INT2STRING(buffer, i) \ _snprintf_s((buffer), \ _scprintf("%d", (i)) + 1, \ _scprintf("%d", (i)) + 1, \ "%d", \ (i)) #define STRCAT(dest, n, src) strcat_s((dest), (n), (src)) #else #define INT2STRING(buffer, i) \ _snprintf((buffer), \ _scprintf("%d", (i)) + 1, \ "%d", \ (i)) #define STRCAT(dest, n, src) strcat((dest), (src)) #endif #else #define INT2STRING(buffer, i) snprintf((buffer), sizeof((buffer)), "%d", (i)) #define STRCAT(dest, n, src) strcat((dest), (src)) #endif #if PY_MAJOR_VERSION >= 3 #define BYTES_FORMAT_STRING "y#" #else #define BYTES_FORMAT_STRING "s#" #endif typedef struct codec_options_t { PyObject* document_class; unsigned char tz_aware; unsigned char uuid_rep; char* unicode_decode_error_handler; PyObject* tzinfo; PyObject* options_obj; unsigned char is_raw_bson; } codec_options_t; /* C API functions */ #define _cbson_buffer_write_bytes_INDEX 0 #define _cbson_buffer_write_bytes_RETURN int #define _cbson_buffer_write_bytes_PROTO (buffer_t buffer, const char* data, int size) #define _cbson_write_dict_INDEX 1 #define _cbson_write_dict_RETURN int #define _cbson_write_dict_PROTO (PyObject* self, buffer_t buffer, PyObject* dict, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) #define _cbson_write_pair_INDEX 2 #define _cbson_write_pair_RETURN int #define _cbson_write_pair_PROTO (PyObject* self, buffer_t buffer, const char* name, int name_length, PyObject* value, unsigned char check_keys, const codec_options_t* options, unsigned char allow_id) #define _cbson_decode_and_write_pair_INDEX 3 #define _cbson_decode_and_write_pair_RETURN int #define _cbson_decode_and_write_pair_PROTO (PyObject* self, buffer_t buffer, PyObject* key, PyObject* value, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) #define _cbson_convert_codec_options_INDEX 4 #define _cbson_convert_codec_options_RETURN int #define _cbson_convert_codec_options_PROTO (PyObject* options_obj, void* p) #define _cbson_destroy_codec_options_INDEX 5 #define _cbson_destroy_codec_options_RETURN void #define _cbson_destroy_codec_options_PROTO (codec_options_t* options) #define _cbson_buffer_write_double_INDEX 6 #define _cbson_buffer_write_double_RETURN int #define _cbson_buffer_write_double_PROTO (buffer_t buffer, double data) #define _cbson_buffer_write_int32_INDEX 7 #define _cbson_buffer_write_int32_RETURN int #define _cbson_buffer_write_int32_PROTO (buffer_t buffer, int32_t data) #define _cbson_buffer_write_int64_INDEX 8 #define _cbson_buffer_write_int64_RETURN int #define _cbson_buffer_write_int64_PROTO (buffer_t buffer, int64_t data) #define _cbson_buffer_write_int32_at_position_INDEX 9 #define _cbson_buffer_write_int32_at_position_RETURN void #define _cbson_buffer_write_int32_at_position_PROTO (buffer_t buffer, int position, int32_t data) /* Total number of C API pointers */ #define _cbson_API_POINTER_COUNT 10 #ifdef _CBSON_MODULE /* This section is used when compiling _cbsonmodule */ static _cbson_buffer_write_bytes_RETURN buffer_write_bytes _cbson_buffer_write_bytes_PROTO; static _cbson_write_dict_RETURN write_dict _cbson_write_dict_PROTO; static _cbson_write_pair_RETURN write_pair _cbson_write_pair_PROTO; static _cbson_decode_and_write_pair_RETURN decode_and_write_pair _cbson_decode_and_write_pair_PROTO; static _cbson_convert_codec_options_RETURN convert_codec_options _cbson_convert_codec_options_PROTO; static _cbson_destroy_codec_options_RETURN destroy_codec_options _cbson_destroy_codec_options_PROTO; static _cbson_buffer_write_double_RETURN buffer_write_double _cbson_buffer_write_double_PROTO; static _cbson_buffer_write_int32_RETURN buffer_write_int32 _cbson_buffer_write_int32_PROTO; static _cbson_buffer_write_int64_RETURN buffer_write_int64 _cbson_buffer_write_int64_PROTO; static _cbson_buffer_write_int32_at_position_RETURN buffer_write_int32_at_position _cbson_buffer_write_int32_at_position_PROTO; #else /* This section is used in modules that use _cbsonmodule's API */ static void **_cbson_API; #define buffer_write_bytes (*(_cbson_buffer_write_bytes_RETURN (*)_cbson_buffer_write_bytes_PROTO) _cbson_API[_cbson_buffer_write_bytes_INDEX]) #define write_dict (*(_cbson_write_dict_RETURN (*)_cbson_write_dict_PROTO) _cbson_API[_cbson_write_dict_INDEX]) #define write_pair (*(_cbson_write_pair_RETURN (*)_cbson_write_pair_PROTO) _cbson_API[_cbson_write_pair_INDEX]) #define decode_and_write_pair (*(_cbson_decode_and_write_pair_RETURN (*)_cbson_decode_and_write_pair_PROTO) _cbson_API[_cbson_decode_and_write_pair_INDEX]) #define convert_codec_options (*(_cbson_convert_codec_options_RETURN (*)_cbson_convert_codec_options_PROTO) _cbson_API[_cbson_convert_codec_options_INDEX]) #define destroy_codec_options (*(_cbson_destroy_codec_options_RETURN (*)_cbson_destroy_codec_options_PROTO) _cbson_API[_cbson_destroy_codec_options_INDEX]) #define buffer_write_double (*(_cbson_buffer_write_double_RETURN (*)_cbson_buffer_write_double_PROTO) _cbson_API[_cbson_buffer_write_double_INDEX]) #define buffer_write_int32 (*(_cbson_buffer_write_int32_RETURN (*)_cbson_buffer_write_int32_PROTO) _cbson_API[_cbson_buffer_write_int32_INDEX]) #define buffer_write_int64 (*(_cbson_buffer_write_int64_RETURN (*)_cbson_buffer_write_int64_PROTO) _cbson_API[_cbson_buffer_write_int64_INDEX]) #define buffer_write_int32_at_position (*(_cbson_buffer_write_int32_at_position_RETURN (*)_cbson_buffer_write_int32_at_position_PROTO) _cbson_API[_cbson_buffer_write_int32_at_position_INDEX]) #define _cbson_IMPORT _cbson_API = (void **)PyCapsule_Import("_cbson._C_API", 0) #endif #endif // _CBSONMODULE_H pymongo-3.6.1/bson/bson-endian.h0000644000076600000240000001471513245617773016771 0ustar shanestaff00000000000000/* * Copyright 2013-2016 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef BSON_ENDIAN_H #define BSON_ENDIAN_H #if defined(__sun) # include #endif #ifdef _MSC_VER # include "bson-stdint-win32.h" # define BSON_INLINE __inline #else # include # define BSON_INLINE __inline__ #endif #define BSON_BIG_ENDIAN 4321 #define BSON_LITTLE_ENDIAN 1234 /* WORDS_BIGENDIAN from pyconfig.h / Python.h */ #ifdef WORDS_BIGENDIAN # define BSON_BYTE_ORDER BSON_BIG_ENDIAN #else # define BSON_BYTE_ORDER BSON_LITTLE_ENDIAN #endif #if defined(__sun) # define BSON_UINT16_SWAP_LE_BE(v) BSWAP_16((uint16_t)v) # define BSON_UINT32_SWAP_LE_BE(v) BSWAP_32((uint32_t)v) # define BSON_UINT64_SWAP_LE_BE(v) BSWAP_64((uint64_t)v) #elif defined(__clang__) && defined(__clang_major__) && defined(__clang_minor__) && \ (__clang_major__ >= 3) && (__clang_minor__ >= 1) # if __has_builtin(__builtin_bswap16) # define BSON_UINT16_SWAP_LE_BE(v) __builtin_bswap16(v) # endif # if __has_builtin(__builtin_bswap32) # define BSON_UINT32_SWAP_LE_BE(v) __builtin_bswap32(v) # endif # if __has_builtin(__builtin_bswap64) # define BSON_UINT64_SWAP_LE_BE(v) __builtin_bswap64(v) # endif #elif defined(__GNUC__) && (__GNUC__ >= 4) # if __GNUC__ >= 4 && defined (__GNUC_MINOR__) && __GNUC_MINOR__ >= 3 # define BSON_UINT32_SWAP_LE_BE(v) __builtin_bswap32 ((uint32_t)v) # define BSON_UINT64_SWAP_LE_BE(v) __builtin_bswap64 ((uint64_t)v) # endif # if __GNUC__ >= 4 && defined (__GNUC_MINOR__) && __GNUC_MINOR__ >= 8 # define BSON_UINT16_SWAP_LE_BE(v) __builtin_bswap16 ((uint32_t)v) # endif #endif #ifndef BSON_UINT16_SWAP_LE_BE # define BSON_UINT16_SWAP_LE_BE(v) __bson_uint16_swap_slow ((uint16_t)v) #endif #ifndef BSON_UINT32_SWAP_LE_BE # define BSON_UINT32_SWAP_LE_BE(v) __bson_uint32_swap_slow ((uint32_t)v) #endif #ifndef BSON_UINT64_SWAP_LE_BE # define BSON_UINT64_SWAP_LE_BE(v) __bson_uint64_swap_slow ((uint64_t)v) #endif #if BSON_BYTE_ORDER == BSON_LITTLE_ENDIAN # define BSON_UINT16_FROM_LE(v) ((uint16_t)v) # define BSON_UINT16_TO_LE(v) ((uint16_t)v) # define BSON_UINT16_FROM_BE(v) BSON_UINT16_SWAP_LE_BE (v) # define BSON_UINT16_TO_BE(v) BSON_UINT16_SWAP_LE_BE (v) # define BSON_UINT32_FROM_LE(v) ((uint32_t)v) # define BSON_UINT32_TO_LE(v) ((uint32_t)v) # define BSON_UINT32_FROM_BE(v) BSON_UINT32_SWAP_LE_BE (v) # define BSON_UINT32_TO_BE(v) BSON_UINT32_SWAP_LE_BE (v) # define BSON_UINT64_FROM_LE(v) ((uint64_t)v) # define BSON_UINT64_TO_LE(v) ((uint64_t)v) # define BSON_UINT64_FROM_BE(v) BSON_UINT64_SWAP_LE_BE (v) # define BSON_UINT64_TO_BE(v) BSON_UINT64_SWAP_LE_BE (v) # define BSON_DOUBLE_FROM_LE(v) ((double)v) # define BSON_DOUBLE_TO_LE(v) ((double)v) #elif BSON_BYTE_ORDER == BSON_BIG_ENDIAN # define BSON_UINT16_FROM_LE(v) BSON_UINT16_SWAP_LE_BE (v) # define BSON_UINT16_TO_LE(v) BSON_UINT16_SWAP_LE_BE (v) # define BSON_UINT16_FROM_BE(v) ((uint16_t)v) # define BSON_UINT16_TO_BE(v) ((uint16_t)v) # define BSON_UINT32_FROM_LE(v) BSON_UINT32_SWAP_LE_BE (v) # define BSON_UINT32_TO_LE(v) BSON_UINT32_SWAP_LE_BE (v) # define BSON_UINT32_FROM_BE(v) ((uint32_t)v) # define BSON_UINT32_TO_BE(v) ((uint32_t)v) # define BSON_UINT64_FROM_LE(v) BSON_UINT64_SWAP_LE_BE (v) # define BSON_UINT64_TO_LE(v) BSON_UINT64_SWAP_LE_BE (v) # define BSON_UINT64_FROM_BE(v) ((uint64_t)v) # define BSON_UINT64_TO_BE(v) ((uint64_t)v) # define BSON_DOUBLE_FROM_LE(v) (__bson_double_swap_slow (v)) # define BSON_DOUBLE_TO_LE(v) (__bson_double_swap_slow (v)) #else # error "The endianness of target architecture is unknown." #endif /* *-------------------------------------------------------------------------- * * __bson_uint16_swap_slow -- * * Fallback endianness conversion for 16-bit integers. * * Returns: * The endian swapped version. * * Side effects: * None. * *-------------------------------------------------------------------------- */ static BSON_INLINE uint16_t __bson_uint16_swap_slow (uint16_t v) /* IN */ { return ((v & 0x00FF) << 8) | ((v & 0xFF00) >> 8); } /* *-------------------------------------------------------------------------- * * __bson_uint32_swap_slow -- * * Fallback endianness conversion for 32-bit integers. * * Returns: * The endian swapped version. * * Side effects: * None. * *-------------------------------------------------------------------------- */ static BSON_INLINE uint32_t __bson_uint32_swap_slow (uint32_t v) /* IN */ { return ((v & 0x000000FFU) << 24) | ((v & 0x0000FF00U) << 8) | ((v & 0x00FF0000U) >> 8) | ((v & 0xFF000000U) >> 24); } /* *-------------------------------------------------------------------------- * * __bson_uint64_swap_slow -- * * Fallback endianness conversion for 64-bit integers. * * Returns: * The endian swapped version. * * Side effects: * None. * *-------------------------------------------------------------------------- */ static BSON_INLINE uint64_t __bson_uint64_swap_slow (uint64_t v) /* IN */ { return ((v & 0x00000000000000FFULL) << 56) | ((v & 0x000000000000FF00ULL) << 40) | ((v & 0x0000000000FF0000ULL) << 24) | ((v & 0x00000000FF000000ULL) << 8) | ((v & 0x000000FF00000000ULL) >> 8) | ((v & 0x0000FF0000000000ULL) >> 24) | ((v & 0x00FF000000000000ULL) >> 40) | ((v & 0xFF00000000000000ULL) >> 56); } /* *-------------------------------------------------------------------------- * * __bson_double_swap_slow -- * * Fallback endianness conversion for double floating point. * * Returns: * The endian swapped version. * * Side effects: * None. * *-------------------------------------------------------------------------- */ static BSON_INLINE double __bson_double_swap_slow (double v) /* IN */ { uint64_t uv; memcpy(&uv, &v, sizeof(v)); uv = BSON_UINT64_SWAP_LE_BE(uv); memcpy(&v, &uv, sizeof(v)); return v; } #endif /* BSON_ENDIAN_H */ pymongo-3.6.1/bson/int64.py0000644000076600000240000000204013245617773015725 0ustar shanestaff00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A BSON wrapper for long (int in python3)""" from bson.py3compat import PY3 if PY3: long = int class Int64(long): """Representation of the BSON int64 type. This is necessary because every integral number is an :class:`int` in Python 3. Small integral numbers are encoded to BSON int32 by default, but Int64 numbers will always be encoded to BSON int64. :Parameters: - `value`: the numeric value to represent """ _type_marker = 18 pymongo-3.6.1/bson/objectid.py0000644000076600000240000002207613245617773016557 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for working with MongoDB `ObjectIds `_. """ import binascii import calendar import datetime import hashlib import os import random import socket import struct import threading import time from bson.errors import InvalidId from bson.py3compat import PY3, bytes_from_hex, string_type, text_type from bson.tz_util import utc def _machine_bytes(): """Get the machine portion of an ObjectId. """ machine_hash = hashlib.md5() if PY3: # gethostname() returns a unicode string in python 3.x # while update() requires a byte string. machine_hash.update(socket.gethostname().encode()) else: # Calling encode() here will fail with non-ascii hostnames machine_hash.update(socket.gethostname()) return machine_hash.digest()[0:3] def _raise_invalid_id(oid): raise InvalidId( "%r is not a valid ObjectId, it must be a 12-byte input" " or a 24-character hex string" % oid) class ObjectId(object): """A MongoDB ObjectId. """ _inc = random.randint(0, 0xFFFFFF) _inc_lock = threading.Lock() _machine_bytes = _machine_bytes() __slots__ = ('__id') _type_marker = 7 def __init__(self, oid=None): """Initialize a new ObjectId. An ObjectId is a 12-byte unique identifier consisting of: - a 4-byte value representing the seconds since the Unix epoch, - a 3-byte machine identifier, - a 2-byte process id, and - a 3-byte counter, starting with a random value. By default, ``ObjectId()`` creates a new unique identifier. The optional parameter `oid` can be an :class:`ObjectId`, or any 12 :class:`bytes` or, in Python 2, any 12-character :class:`str`. For example, the 12 bytes b'foo-bar-quux' do not follow the ObjectId specification but they are acceptable input:: >>> ObjectId(b'foo-bar-quux') ObjectId('666f6f2d6261722d71757578') `oid` can also be a :class:`unicode` or :class:`str` of 24 hex digits:: >>> ObjectId('0123456789ab0123456789ab') ObjectId('0123456789ab0123456789ab') >>> >>> # A u-prefixed unicode literal: >>> ObjectId(u'0123456789ab0123456789ab') ObjectId('0123456789ab0123456789ab') Raises :class:`~bson.errors.InvalidId` if `oid` is not 12 bytes nor 24 hex digits, or :class:`TypeError` if `oid` is not an accepted type. :Parameters: - `oid` (optional): a valid ObjectId. .. mongodoc:: objectids """ if oid is None: self.__generate() elif isinstance(oid, bytes) and len(oid) == 12: self.__id = oid else: self.__validate(oid) @classmethod def from_datetime(cls, generation_time): """Create a dummy ObjectId instance with a specific generation time. This method is useful for doing range queries on a field containing :class:`ObjectId` instances. .. warning:: It is not safe to insert a document containing an ObjectId generated using this method. This method deliberately eliminates the uniqueness guarantee that ObjectIds generally provide. ObjectIds generated with this method should be used exclusively in queries. `generation_time` will be converted to UTC. Naive datetime instances will be treated as though they already contain UTC. An example using this helper to get documents where ``"_id"`` was generated before January 1, 2010 would be: >>> gen_time = datetime.datetime(2010, 1, 1) >>> dummy_id = ObjectId.from_datetime(gen_time) >>> result = collection.find({"_id": {"$lt": dummy_id}}) :Parameters: - `generation_time`: :class:`~datetime.datetime` to be used as the generation time for the resulting ObjectId. """ if generation_time.utcoffset() is not None: generation_time = generation_time - generation_time.utcoffset() timestamp = calendar.timegm(generation_time.timetuple()) oid = struct.pack( ">i", int(timestamp)) + b"\x00\x00\x00\x00\x00\x00\x00\x00" return cls(oid) @classmethod def is_valid(cls, oid): """Checks if a `oid` string is valid or not. :Parameters: - `oid`: the object id to validate .. versionadded:: 2.3 """ if not oid: return False try: ObjectId(oid) return True except (InvalidId, TypeError): return False def __generate(self): """Generate a new value for this ObjectId. """ # 4 bytes current time oid = struct.pack(">i", int(time.time())) # 3 bytes machine oid += ObjectId._machine_bytes # 2 bytes pid oid += struct.pack(">H", os.getpid() % 0xFFFF) # 3 bytes inc with ObjectId._inc_lock: oid += struct.pack(">i", ObjectId._inc)[1:4] ObjectId._inc = (ObjectId._inc + 1) % 0xFFFFFF self.__id = oid def __validate(self, oid): """Validate and use the given id for this ObjectId. Raises TypeError if id is not an instance of (:class:`basestring` (:class:`str` or :class:`bytes` in python 3), ObjectId) and InvalidId if it is not a valid ObjectId. :Parameters: - `oid`: a valid ObjectId """ if isinstance(oid, ObjectId): self.__id = oid.binary # bytes or unicode in python 2, str in python 3 elif isinstance(oid, string_type): if len(oid) == 24: try: self.__id = bytes_from_hex(oid) except (TypeError, ValueError): _raise_invalid_id(oid) else: _raise_invalid_id(oid) else: raise TypeError("id must be an instance of (bytes, %s, ObjectId), " "not %s" % (text_type.__name__, type(oid))) @property def binary(self): """12-byte binary representation of this ObjectId. """ return self.__id @property def generation_time(self): """A :class:`datetime.datetime` instance representing the time of generation for this :class:`ObjectId`. The :class:`datetime.datetime` is timezone aware, and represents the generation time in UTC. It is precise to the second. """ timestamp = struct.unpack(">i", self.__id[0:4])[0] return datetime.datetime.fromtimestamp(timestamp, utc) def __getstate__(self): """return value of object for pickling. needed explicitly because __slots__() defined. """ return self.__id def __setstate__(self, value): """explicit state set from pickling """ # Provide backwards compatability with OIDs # pickled with pymongo-1.9 or older. if isinstance(value, dict): oid = value["_ObjectId__id"] else: oid = value # ObjectIds pickled in python 2.x used `str` for __id. # In python 3.x this has to be converted to `bytes` # by encoding latin-1. if PY3 and isinstance(oid, text_type): self.__id = oid.encode('latin-1') else: self.__id = oid def __str__(self): if PY3: return binascii.hexlify(self.__id).decode() return binascii.hexlify(self.__id) def __repr__(self): return "ObjectId('%s')" % (str(self),) def __eq__(self, other): if isinstance(other, ObjectId): return self.__id == other.binary return NotImplemented def __ne__(self, other): if isinstance(other, ObjectId): return self.__id != other.binary return NotImplemented def __lt__(self, other): if isinstance(other, ObjectId): return self.__id < other.binary return NotImplemented def __le__(self, other): if isinstance(other, ObjectId): return self.__id <= other.binary return NotImplemented def __gt__(self, other): if isinstance(other, ObjectId): return self.__id > other.binary return NotImplemented def __ge__(self, other): if isinstance(other, ObjectId): return self.__id >= other.binary return NotImplemented def __hash__(self): """Get a hash value for this :class:`ObjectId`.""" return hash(self.__id) pymongo-3.6.1/bson/dbref.py0000644000076600000240000001117513245617773016054 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for manipulating DBRefs (references to MongoDB documents).""" from copy import deepcopy from bson.py3compat import iteritems, string_type from bson.son import SON class DBRef(object): """A reference to a document stored in MongoDB. """ # DBRef isn't actually a BSON "type" so this number was arbitrarily chosen. _type_marker = 100 def __init__(self, collection, id, database=None, _extra={}, **kwargs): """Initialize a new :class:`DBRef`. Raises :class:`TypeError` if `collection` or `database` is not an instance of :class:`basestring` (:class:`str` in python 3). `database` is optional and allows references to documents to work across databases. Any additional keyword arguments will create additional fields in the resultant embedded document. :Parameters: - `collection`: name of the collection the document is stored in - `id`: the value of the document's ``"_id"`` field - `database` (optional): name of the database to reference - `**kwargs` (optional): additional keyword arguments will create additional, custom fields .. mongodoc:: dbrefs """ if not isinstance(collection, string_type): raise TypeError("collection must be an " "instance of %s" % string_type.__name__) if database is not None and not isinstance(database, string_type): raise TypeError("database must be an " "instance of %s" % string_type.__name__) self.__collection = collection self.__id = id self.__database = database kwargs.update(_extra) self.__kwargs = kwargs @property def collection(self): """Get the name of this DBRef's collection as unicode. """ return self.__collection @property def id(self): """Get this DBRef's _id. """ return self.__id @property def database(self): """Get the name of this DBRef's database. Returns None if this DBRef doesn't specify a database. """ return self.__database def __getattr__(self, key): try: return self.__kwargs[key] except KeyError: raise AttributeError(key) # Have to provide __setstate__ to avoid # infinite recursion since we override # __getattr__. def __setstate__(self, state): self.__dict__.update(state) def as_doc(self): """Get the SON document representation of this DBRef. Generally not needed by application developers """ doc = SON([("$ref", self.collection), ("$id", self.id)]) if self.database is not None: doc["$db"] = self.database doc.update(self.__kwargs) return doc def __repr__(self): extra = "".join([", %s=%r" % (k, v) for k, v in iteritems(self.__kwargs)]) if self.database is None: return "DBRef(%r, %r%s)" % (self.collection, self.id, extra) return "DBRef(%r, %r, %r%s)" % (self.collection, self.id, self.database, extra) def __eq__(self, other): if isinstance(other, DBRef): us = (self.__database, self.__collection, self.__id, self.__kwargs) them = (other.__database, other.__collection, other.__id, other.__kwargs) return us == them return NotImplemented def __ne__(self, other): return not self == other def __hash__(self): """Get a hash value for this :class:`DBRef`.""" return hash((self.__collection, self.__id, self.__database, tuple(sorted(self.__kwargs.items())))) def __deepcopy__(self, memo): """Support function for `copy.deepcopy()`.""" return DBRef(deepcopy(self.__collection, memo), deepcopy(self.__id, memo), deepcopy(self.__database, memo), deepcopy(self.__kwargs, memo)) pymongo-3.6.1/bson/code.py0000644000076600000240000000644013245621354015671 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing JavaScript code in BSON. """ from bson.py3compat import abc, string_type, PY3, text_type class Code(str): """BSON's JavaScript code type. Raises :class:`TypeError` if `code` is not an instance of :class:`basestring` (:class:`str` in python 3) or `scope` is not ``None`` or an instance of :class:`dict`. Scope variables can be set by passing a dictionary as the `scope` argument or by using keyword arguments. If a variable is set as a keyword argument it will override any setting for that variable in the `scope` dictionary. :Parameters: - `code`: A string containing JavaScript code to be evaluated or another instance of Code. In the latter case, the scope of `code` becomes this Code's :attr:`scope`. - `scope` (optional): dictionary representing the scope in which `code` should be evaluated - a mapping from identifiers (as strings) to values. Defaults to ``None``. This is applied after any scope associated with a given `code` above. - `**kwargs` (optional): scope variables can also be passed as keyword arguments. These are applied after `scope` and `code`. .. versionchanged:: 3.4 The default value for :attr:`scope` is ``None`` instead of ``{}``. """ _type_marker = 13 def __new__(cls, code, scope=None, **kwargs): if not isinstance(code, string_type): raise TypeError("code must be an " "instance of %s" % (string_type.__name__)) if not PY3 and isinstance(code, text_type): self = str.__new__(cls, code.encode('utf8')) else: self = str.__new__(cls, code) try: self.__scope = code.scope except AttributeError: self.__scope = None if scope is not None: if not isinstance(scope, abc.Mapping): raise TypeError("scope must be an instance of dict") if self.__scope is not None: self.__scope.update(scope) else: self.__scope = scope if kwargs: if self.__scope is not None: self.__scope.update(kwargs) else: self.__scope = kwargs return self @property def scope(self): """Scope dictionary for this instance or ``None``. """ return self.__scope def __repr__(self): return "Code(%s, %r)" % (str.__repr__(self), self.__scope) def __eq__(self, other): if isinstance(other, Code): return (self.__scope, str(self)) == (other.__scope, str(other)) return False __hash__ = None def __ne__(self, other): return not self == other pymongo-3.6.1/bson/__init__.py0000644000076600000240000011235013245621354016514 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """BSON (Binary JSON) encoding and decoding. The mapping from Python types to BSON types is as follows: ======================================= ============= =================== Python Type BSON Type Supported Direction ======================================= ============= =================== None null both bool boolean both int [#int]_ int32 / int64 py -> bson long int64 py -> bson `bson.int64.Int64` int64 both float number (real) both string string py -> bson unicode string both list array both dict / `SON` object both datetime.datetime [#dt]_ [#dt2]_ date both `bson.regex.Regex` regex both compiled re [#re]_ regex py -> bson `bson.binary.Binary` binary both `bson.objectid.ObjectId` oid both `bson.dbref.DBRef` dbref both None undefined bson -> py unicode code bson -> py `bson.code.Code` code py -> bson unicode symbol bson -> py bytes (Python 3) [#bytes]_ binary both ======================================= ============= =================== Note that, when using Python 2.x, to save binary data it must be wrapped as an instance of `bson.binary.Binary`. Otherwise it will be saved as a BSON string and retrieved as unicode. Users of Python 3.x can use the Python bytes type. .. [#int] A Python int will be saved as a BSON int32 or BSON int64 depending on its size. A BSON int32 will always decode to a Python int. A BSON int64 will always decode to a :class:`~bson.int64.Int64`. .. [#dt] datetime.datetime instances will be rounded to the nearest millisecond when saved .. [#dt2] all datetime.datetime instances are treated as *naive*. clients should always use UTC. .. [#re] :class:`~bson.regex.Regex` instances and regular expression objects from ``re.compile()`` are both saved as BSON regular expressions. BSON regular expressions are decoded as :class:`~bson.regex.Regex` instances. .. [#bytes] The bytes type from Python 3.x is encoded as BSON binary with subtype 0. In Python 3.x it will be decoded back to bytes. In Python 2.x it will be decoded to an instance of :class:`~bson.binary.Binary` with subtype 0. """ import calendar import datetime import itertools import re import struct import sys import uuid from codecs import (utf_8_decode as _utf_8_decode, utf_8_encode as _utf_8_encode) from bson.binary import (Binary, OLD_UUID_SUBTYPE, JAVA_LEGACY, CSHARP_LEGACY, UUIDLegacy) from bson.code import Code from bson.codec_options import ( CodecOptions, DEFAULT_CODEC_OPTIONS, _raw_document_class) from bson.dbref import DBRef from bson.decimal128 import Decimal128 from bson.errors import (InvalidBSON, InvalidDocument, InvalidStringData) from bson.int64 import Int64 from bson.max_key import MaxKey from bson.min_key import MinKey from bson.objectid import ObjectId from bson.py3compat import (abc, b, PY3, iteritems, text_type, string_type, reraise) from bson.regex import Regex from bson.son import SON, RE_TYPE from bson.timestamp import Timestamp from bson.tz_util import utc try: from bson import _cbson _USE_C = True except ImportError: _USE_C = False EPOCH_AWARE = datetime.datetime.fromtimestamp(0, utc) EPOCH_NAIVE = datetime.datetime.utcfromtimestamp(0) BSONNUM = b"\x01" # Floating point BSONSTR = b"\x02" # UTF-8 string BSONOBJ = b"\x03" # Embedded document BSONARR = b"\x04" # Array BSONBIN = b"\x05" # Binary BSONUND = b"\x06" # Undefined BSONOID = b"\x07" # ObjectId BSONBOO = b"\x08" # Boolean BSONDAT = b"\x09" # UTC Datetime BSONNUL = b"\x0A" # Null BSONRGX = b"\x0B" # Regex BSONREF = b"\x0C" # DBRef BSONCOD = b"\x0D" # Javascript code BSONSYM = b"\x0E" # Symbol BSONCWS = b"\x0F" # Javascript code with scope BSONINT = b"\x10" # 32bit int BSONTIM = b"\x11" # Timestamp BSONLON = b"\x12" # 64bit int BSONDEC = b"\x13" # Decimal128 BSONMIN = b"\xFF" # Min key BSONMAX = b"\x7F" # Max key _UNPACK_FLOAT = struct.Struct("= obj_end: raise InvalidBSON("invalid object length") if _raw_document_class(opts.document_class): return (opts.document_class(data[position:end + 1], opts), position + obj_size) obj = _elements_to_dict(data, position + 4, end, opts) position += obj_size if "$ref" in obj: return (DBRef(obj.pop("$ref"), obj.pop("$id", None), obj.pop("$db", None), obj), position) return obj, position def _get_array(data, position, obj_end, opts, element_name): """Decode a BSON array to python list.""" size = _UNPACK_INT(data[position:position + 4])[0] end = position + size - 1 if data[end:end + 1] != b"\x00": raise InvalidBSON("bad eoo") position += 4 end -= 1 result = [] # Avoid doing global and attibute lookups in the loop. append = result.append index = data.index getter = _ELEMENT_GETTER while position < end: element_type = data[position:position + 1] # Just skip the keys. position = index(b'\x00', position) + 1 try: value, position = getter[element_type]( data, position, obj_end, opts, element_name) except KeyError: _raise_unknown_type(element_type, element_name) append(value) if position != end + 1: raise InvalidBSON('bad array length') return result, position + 1 def _get_binary(data, position, obj_end, opts, dummy1): """Decode a BSON binary to bson.binary.Binary or python UUID.""" length, subtype = _UNPACK_LENGTH_SUBTYPE(data[position:position + 5]) position += 5 if subtype == 2: length2 = _UNPACK_INT(data[position:position + 4])[0] position += 4 if length2 != length - 4: raise InvalidBSON("invalid binary (st 2) - lengths don't match!") length = length2 end = position + length if length < 0 or end > obj_end: raise InvalidBSON('bad binary object length') if subtype == 3: # Java Legacy uuid_representation = opts.uuid_representation if uuid_representation == JAVA_LEGACY: java = data[position:end] value = uuid.UUID(bytes=java[0:8][::-1] + java[8:16][::-1]) # C# legacy elif uuid_representation == CSHARP_LEGACY: value = uuid.UUID(bytes_le=data[position:end]) # Python else: value = uuid.UUID(bytes=data[position:end]) return value, end if subtype == 4: return uuid.UUID(bytes=data[position:end]), end # Python3 special case. Decode subtype 0 to 'bytes'. if PY3 and subtype == 0: value = data[position:end] else: value = Binary(data[position:end], subtype) return value, end def _get_oid(data, position, dummy0, dummy1, dummy2): """Decode a BSON ObjectId to bson.objectid.ObjectId.""" end = position + 12 return ObjectId(data[position:end]), end def _get_boolean(data, position, dummy0, dummy1, dummy2): """Decode a BSON true/false to python True/False.""" end = position + 1 boolean_byte = data[position:end] if boolean_byte == b'\x00': return False, end elif boolean_byte == b'\x01': return True, end raise InvalidBSON('invalid boolean value: %r' % boolean_byte) def _get_date(data, position, dummy0, opts, dummy1): """Decode a BSON datetime to python datetime.datetime.""" end = position + 8 millis = _UNPACK_LONG(data[position:end])[0] return _millis_to_datetime(millis, opts), end def _get_code(data, position, obj_end, opts, element_name): """Decode a BSON code to bson.code.Code.""" code, position = _get_string(data, position, obj_end, opts, element_name) return Code(code), position def _get_code_w_scope(data, position, obj_end, opts, element_name): """Decode a BSON code_w_scope to bson.code.Code.""" code_end = position + _UNPACK_INT(data[position:position + 4])[0] code, position = _get_string( data, position + 4, code_end, opts, element_name) scope, position = _get_object(data, position, code_end, opts, element_name) if position != code_end: raise InvalidBSON('scope outside of javascript code boundaries') return Code(code, scope), position def _get_regex(data, position, dummy0, opts, dummy1): """Decode a BSON regex to bson.regex.Regex or a python pattern object.""" pattern, position = _get_c_string(data, position, opts) bson_flags, position = _get_c_string(data, position, opts) bson_re = Regex(pattern, bson_flags) return bson_re, position def _get_ref(data, position, obj_end, opts, element_name): """Decode (deprecated) BSON DBPointer to bson.dbref.DBRef.""" collection, position = _get_string( data, position, obj_end, opts, element_name) oid, position = _get_oid(data, position, obj_end, opts, element_name) return DBRef(collection, oid), position def _get_timestamp(data, position, dummy0, dummy1, dummy2): """Decode a BSON timestamp to bson.timestamp.Timestamp.""" end = position + 8 inc, timestamp = _UNPACK_TIMESTAMP(data[position:end]) return Timestamp(timestamp, inc), end def _get_int64(data, position, dummy0, dummy1, dummy2): """Decode a BSON int64 to bson.int64.Int64.""" end = position + 8 return Int64(_UNPACK_LONG(data[position:end])[0]), end def _get_decimal128(data, position, dummy0, dummy1, dummy2): """Decode a BSON decimal128 to bson.decimal128.Decimal128.""" end = position + 16 return Decimal128.from_bid(data[position:end]), end # Each decoder function's signature is: # - data: bytes # - position: int, beginning of object in 'data' to decode # - obj_end: int, end of object to decode in 'data' if variable-length type # - opts: a CodecOptions _ELEMENT_GETTER = { BSONNUM: _get_float, BSONSTR: _get_string, BSONOBJ: _get_object, BSONARR: _get_array, BSONBIN: _get_binary, BSONUND: lambda v, w, x, y, z: (None, w), # Deprecated undefined BSONOID: _get_oid, BSONBOO: _get_boolean, BSONDAT: _get_date, BSONNUL: lambda v, w, x, y, z: (None, w), BSONRGX: _get_regex, BSONREF: _get_ref, # Deprecated DBPointer BSONCOD: _get_code, BSONSYM: _get_string, # Deprecated symbol BSONCWS: _get_code_w_scope, BSONINT: _get_int, BSONTIM: _get_timestamp, BSONLON: _get_int64, BSONDEC: _get_decimal128, BSONMIN: lambda v, w, x, y, z: (MinKey(), w), BSONMAX: lambda v, w, x, y, z: (MaxKey(), w)} def _element_to_dict(data, position, obj_end, opts): """Decode a single key, value pair.""" element_type = data[position:position + 1] position += 1 element_name, position = _get_c_string(data, position, opts) try: value, position = _ELEMENT_GETTER[element_type](data, position, obj_end, opts, element_name) except KeyError: _raise_unknown_type(element_type, element_name) return element_name, value, position if _USE_C: _element_to_dict = _cbson._element_to_dict def _iterate_elements(data, position, obj_end, opts): end = obj_end - 1 while position < end: (key, value, position) = _element_to_dict(data, position, obj_end, opts) yield key, value, position def _elements_to_dict(data, position, obj_end, opts): """Decode a BSON document.""" result = opts.document_class() pos = position for key, value, pos in _iterate_elements(data, position, obj_end, opts): result[key] = value if pos != obj_end: raise InvalidBSON('bad object or element length') return result def _bson_to_dict(data, opts): """Decode a BSON string to document_class.""" try: obj_size = _UNPACK_INT(data[:4])[0] except struct.error as exc: raise InvalidBSON(str(exc)) if obj_size != len(data): raise InvalidBSON("invalid object size") if data[obj_size - 1:obj_size] != b"\x00": raise InvalidBSON("bad eoo") try: if _raw_document_class(opts.document_class): return opts.document_class(data, opts) return _elements_to_dict(data, 4, obj_size - 1, opts) except InvalidBSON: raise except Exception: # Change exception type to InvalidBSON but preserve traceback. _, exc_value, exc_tb = sys.exc_info() reraise(InvalidBSON, exc_value, exc_tb) if _USE_C: _bson_to_dict = _cbson._bson_to_dict _PACK_FLOAT = struct.Struct(">> import collections # From Python standard library. >>> import bson >>> from bson.codec_options import CodecOptions >>> data = bson.BSON.encode({'a': 1}) >>> decoded_doc = bson.BSON.decode(data) >>> options = CodecOptions(document_class=collections.OrderedDict) >>> decoded_doc = bson.BSON.decode(data, codec_options=options) >>> type(decoded_doc) :Parameters: - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. .. versionchanged:: 3.0 Removed `compile_re` option: PyMongo now always represents BSON regular expressions as :class:`~bson.regex.Regex` objects. Use :meth:`~bson.regex.Regex.try_compile` to attempt to convert from a BSON regular expression to a Python regular expression object. Replaced `as_class`, `tz_aware`, and `uuid_subtype` options with `codec_options`. .. versionchanged:: 2.7 Added `compile_re` option. If set to False, PyMongo represented BSON regular expressions as :class:`~bson.regex.Regex` objects instead of attempting to compile BSON regular expressions as Python native regular expressions, thus preventing errors for some incompatible patterns, see `PYTHON-500`_. .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500 """ if not isinstance(codec_options, CodecOptions): raise _CODEC_OPTIONS_TYPE_ERROR return _bson_to_dict(self, codec_options) def has_c(): """Is the C extension installed? """ return _USE_C pymongo-3.6.1/bson/tz_util.py0000644000076600000240000000275613156613521016455 0ustar shanestaff00000000000000# Copyright 2010-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Timezone related utilities for BSON.""" from datetime import (timedelta, tzinfo) ZERO = timedelta(0) class FixedOffset(tzinfo): """Fixed offset timezone, in minutes east from UTC. Implementation based from the Python `standard library documentation `_. Defining __getinitargs__ enables pickling / copying. """ def __init__(self, offset, name): if isinstance(offset, timedelta): self.__offset = offset else: self.__offset = timedelta(minutes=offset) self.__name = name def __getinitargs__(self): return self.__offset, self.__name def utcoffset(self, dt): return self.__offset def tzname(self, dt): return self.__name def dst(self, dt): return ZERO utc = FixedOffset(0, "UTC") """Fixed offset timezone representing UTC.""" pymongo-3.6.1/bson/json_util.py0000644000076600000240000007771513245621354017002 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for using Python's :mod:`json` module with BSON documents. This module provides two helper methods `dumps` and `loads` that wrap the native :mod:`json` methods and provide explicit BSON conversion to and from JSON. :class:`~bson.json_util.JSONOptions` provides a way to control how JSON is emitted and parsed, with the default being the legacy PyMongo format. :mod:`~bson.json_util` can also generate Canonical or Relaxed `Extended JSON`_ when :const:`CANONICAL_JSON_OPTIONS` or :const:`RELAXED_JSON_OPTIONS` is provided, respectively. .. _Extended JSON: https://github.com/mongodb/specifications/blob/master/source/extended-json.rst Example usage (deserialization): .. doctest:: >>> from bson.json_util import loads >>> loads('[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$scope": {}, "$code": "function x() { return 1; }"}}, {"bin": {"$type": "80", "$binary": "AQIDBA=="}}]') [{u'foo': [1, 2]}, {u'bar': {u'hello': u'world'}}, {u'code': Code('function x() { return 1; }', {})}, {u'bin': Binary('...', 128)}] Example usage (serialization): .. doctest:: >>> from bson import Binary, Code >>> from bson.json_util import dumps >>> dumps([{'foo': [1, 2]}, ... {'bar': {'hello': 'world'}}, ... {'code': Code("function x() { return 1; }", {})}, ... {'bin': Binary(b"\x01\x02\x03\x04")}]) '[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$code": "function x() { return 1; }", "$scope": {}}}, {"bin": {"$binary": "AQIDBA==", "$type": "00"}}]' Example usage (with :const:`CANONICAL_JSON_OPTIONS`): .. doctest:: >>> from bson import Binary, Code >>> from bson.json_util import dumps, CANONICAL_JSON_OPTIONS >>> dumps([{'foo': [1, 2]}, ... {'bar': {'hello': 'world'}}, ... {'code': Code("function x() { return 1; }")}, ... {'bin': Binary(b"\x01\x02\x03\x04")}], ... json_options=CANONICAL_JSON_OPTIONS) '[{"foo": [{"$numberInt": "1"}, {"$numberInt": "2"}]}, {"bar": {"hello": "world"}}, {"code": {"$code": "function x() { return 1; }"}}, {"bin": {"$binary": {"base64": "AQIDBA==", "subType": "00"}}}]' Example usage (with :const:`RELAXED_JSON_OPTIONS`): .. doctest:: >>> from bson import Binary, Code >>> from bson.json_util import dumps, RELAXED_JSON_OPTIONS >>> dumps([{'foo': [1, 2]}, ... {'bar': {'hello': 'world'}}, ... {'code': Code("function x() { return 1; }")}, ... {'bin': Binary(b"\x01\x02\x03\x04")}], ... json_options=RELAXED_JSON_OPTIONS) '[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$code": "function x() { return 1; }"}}, {"bin": {"$binary": {"base64": "AQIDBA==", "subType": "00"}}}]' Alternatively, you can manually pass the `default` to :func:`json.dumps`. It won't handle :class:`~bson.binary.Binary` and :class:`~bson.code.Code` instances (as they are extended strings you can't provide custom defaults), but it will be faster as there is less recursion. .. note:: If your application does not need the flexibility offered by :class:`JSONOptions` and spends a large amount of time in the `json_util` module, look to `python-bsonjs `_ for a nice performance improvement. `python-bsonjs` is a fast BSON to MongoDB Extended JSON converter for Python built on top of `libbson `_. `python-bsonjs` works best with PyMongo when using :class:`~bson.raw_bson.RawBSONDocument`. .. versionchanged:: 2.8 The output format for :class:`~bson.timestamp.Timestamp` has changed from '{"t": , "i": }' to '{"$timestamp": {"t": , "i": }}'. This new format will be decoded to an instance of :class:`~bson.timestamp.Timestamp`. The old format will continue to be decoded to a python dict as before. Encoding to the old format is no longer supported as it was never correct and loses type information. Added support for $numberLong and $undefined - new in MongoDB 2.6 - and parsing $date in ISO-8601 format. .. versionchanged:: 2.7 Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef instances. .. versionchanged:: 2.3 Added dumps and loads helpers to automatically handle conversion to and from json and supports :class:`~bson.binary.Binary` and :class:`~bson.code.Code` """ import base64 import datetime import math import re import sys import uuid if sys.version_info[:2] == (2, 6): # In Python 2.6, json does not include object_pairs_hook. Use simplejson # instead. try: import simplejson as json except ImportError: import json else: import json from pymongo.errors import ConfigurationError import bson from bson import EPOCH_AWARE, EPOCH_NAIVE, RE_TYPE, SON from bson.binary import (Binary, JAVA_LEGACY, CSHARP_LEGACY, OLD_UUID_SUBTYPE, UUID_SUBTYPE) from bson.code import Code from bson.codec_options import CodecOptions from bson.dbref import DBRef from bson.decimal128 import Decimal128 from bson.int64 import Int64 from bson.max_key import MaxKey from bson.min_key import MinKey from bson.objectid import ObjectId from bson.py3compat import (PY3, iteritems, integer_types, string_type, text_type) from bson.regex import Regex from bson.timestamp import Timestamp from bson.tz_util import utc try: json.loads("{}", object_pairs_hook=dict) _HAS_OBJECT_PAIRS_HOOK = True except TypeError: _HAS_OBJECT_PAIRS_HOOK = False _RE_OPT_TABLE = { "i": re.I, "l": re.L, "m": re.M, "s": re.S, "u": re.U, "x": re.X, } # Dollar-prefixed keys which may appear in DBRefs. _DBREF_KEYS = frozenset(['$id', '$ref', '$db']) class DatetimeRepresentation: LEGACY = 0 """Legacy MongoDB Extended JSON datetime representation. :class:`datetime.datetime` instances will be encoded to JSON in the format `{"$date": }`, where `dateAsMilliseconds` is a 64-bit signed integer giving the number of milliseconds since the Unix epoch UTC. This was the default encoding before PyMongo version 3.4. .. versionadded:: 3.4 """ NUMBERLONG = 1 """NumberLong datetime representation. :class:`datetime.datetime` instances will be encoded to JSON in the format `{"$date": {"$numberLong": ""}}`, where `dateAsMilliseconds` is the string representation of a 64-bit signed integer giving the number of milliseconds since the Unix epoch UTC. .. versionadded:: 3.4 """ ISO8601 = 2 """ISO-8601 datetime representation. :class:`datetime.datetime` instances greater than or equal to the Unix epoch UTC will be encoded to JSON in the format `{"$date": ""}`. :class:`datetime.datetime` instances before the Unix epoch UTC will be encoded as if the datetime representation is :const:`~DatetimeRepresentation.NUMBERLONG`. .. versionadded:: 3.4 """ class JSONMode: LEGACY = 0 """Legacy Extended JSON representation. In this mode, :func:`~bson.json_util.dumps` produces PyMongo's legacy non-standard JSON output. Consider using :const:`~bson.json_util.JSONMode.RELAXED` or :const:`~bson.json_util.JSONMode.CANONICAL` instead. .. versionadded:: 3.5 """ RELAXED = 1 """Relaxed Extended JSON representation. In this mode, :func:`~bson.json_util.dumps` produces Relaxed Extended JSON, a mostly JSON-like format. Consider using this for things like a web API, where one is sending a document (or a projection of a document) that only uses ordinary JSON type primitives. In particular, the ``int``, :class:`~bson.int64.Int64`, and ``float`` numeric types are represented in the native JSON number format. This output is also the most human readable and is useful for debugging and documentation. .. seealso:: The specification for Relaxed `Extended JSON`_. .. versionadded:: 3.5 """ CANONICAL = 2 """Canonical Extended JSON representation. In this mode, :func:`~bson.json_util.dumps` produces Canonical Extended JSON, a type preserving format. Consider using this for things like testing, where one has to precisely specify expected types in JSON. In particular, the ``int``, :class:`~bson.int64.Int64`, and ``float`` numeric types are encoded with type wrappers. .. seealso:: The specification for Canonical `Extended JSON`_. .. versionadded:: 3.5 """ class JSONOptions(CodecOptions): """Encapsulates JSON options for :func:`dumps` and :func:`loads`. Raises :exc:`~pymongo.errors.ConfigurationError` on Python 2.6 if `simplejson >= 2.1.0 `_ is not installed and document_class is not the default (:class:`dict`). :Parameters: - `strict_number_long`: If ``True``, :class:`~bson.int64.Int64` objects are encoded to MongoDB Extended JSON's *Strict mode* type `NumberLong`, ie ``'{"$numberLong": "" }'``. Otherwise they will be encoded as an `int`. Defaults to ``False``. - `datetime_representation`: The representation to use when encoding instances of :class:`datetime.datetime`. Defaults to :const:`~DatetimeRepresentation.LEGACY`. - `strict_uuid`: If ``True``, :class:`uuid.UUID` object are encoded to MongoDB Extended JSON's *Strict mode* type `Binary`. Otherwise it will be encoded as ``'{"$uuid": "" }'``. Defaults to ``False``. - `json_mode`: The :class:`JSONMode` to use when encoding BSON types to Extended JSON. Defaults to :const:`~JSONMode.LEGACY`. - `document_class`: BSON documents returned by :func:`loads` will be decoded to an instance of this class. Must be a subclass of :class:`collections.MutableMapping`. Defaults to :class:`dict`. - `uuid_representation`: The BSON representation to use when encoding and decoding instances of :class:`uuid.UUID`. Defaults to :const:`~bson.binary.PYTHON_LEGACY`. - `tz_aware`: If ``True``, MongoDB Extended JSON's *Strict mode* type `Date` will be decoded to timezone aware instances of :class:`datetime.datetime`. Otherwise they will be naive. Defaults to ``True``. - `tzinfo`: A :class:`datetime.tzinfo` subclass that specifies the timezone from which :class:`~datetime.datetime` objects should be decoded. Defaults to :const:`~bson.tz_util.utc`. - `args`: arguments to :class:`~bson.codec_options.CodecOptions` - `kwargs`: arguments to :class:`~bson.codec_options.CodecOptions` .. seealso:: The specification for Relaxed and Canonical `Extended JSON`_. .. versionadded:: 3.4 .. versionchanged:: 3.5 Accepts the optional parameter `json_mode`. """ def __new__(cls, strict_number_long=False, datetime_representation=DatetimeRepresentation.LEGACY, strict_uuid=False, json_mode=JSONMode.LEGACY, *args, **kwargs): kwargs["tz_aware"] = kwargs.get("tz_aware", True) if kwargs["tz_aware"]: kwargs["tzinfo"] = kwargs.get("tzinfo", utc) if datetime_representation not in (DatetimeRepresentation.LEGACY, DatetimeRepresentation.NUMBERLONG, DatetimeRepresentation.ISO8601): raise ConfigurationError( "JSONOptions.datetime_representation must be one of LEGACY, " "NUMBERLONG, or ISO8601 from DatetimeRepresentation.") self = super(JSONOptions, cls).__new__(cls, *args, **kwargs) if not _HAS_OBJECT_PAIRS_HOOK and self.document_class != dict: raise ConfigurationError( "Support for JSONOptions.document_class on Python 2.6 " "requires simplejson >= 2.1.0" "(https://pypi.python.org/pypi/simplejson) to be installed.") if json_mode not in (JSONMode.LEGACY, JSONMode.RELAXED, JSONMode.CANONICAL): raise ConfigurationError( "JSONOptions.json_mode must be one of LEGACY, RELAXED, " "or CANONICAL from JSONMode.") self.json_mode = json_mode if self.json_mode == JSONMode.RELAXED: self.strict_number_long = False self.datetime_representation = DatetimeRepresentation.ISO8601 self.strict_uuid = True elif self.json_mode == JSONMode.CANONICAL: self.strict_number_long = True self.datetime_representation = DatetimeRepresentation.NUMBERLONG self.strict_uuid = True else: self.strict_number_long = strict_number_long self.datetime_representation = datetime_representation self.strict_uuid = strict_uuid return self def _arguments_repr(self): return ('strict_number_long=%r, ' 'datetime_representation=%r, ' 'strict_uuid=%r, json_mode=%r, %s' % ( self.strict_number_long, self.datetime_representation, self.strict_uuid, self.json_mode, super(JSONOptions, self)._arguments_repr())) LEGACY_JSON_OPTIONS = JSONOptions(json_mode=JSONMode.LEGACY) """:class:`JSONOptions` for encoding to PyMongo's legacy JSON format. .. seealso:: The documentation for :const:`bson.json_util.JSONMode.LEGACY`. .. versionadded:: 3.5 """ DEFAULT_JSON_OPTIONS = LEGACY_JSON_OPTIONS """The default :class:`JSONOptions` for JSON encoding/decoding. The same as :const:`LEGACY_JSON_OPTIONS`. This will change to :const:`RELAXED_JSON_OPTIONS` in a future release. .. versionadded:: 3.4 """ CANONICAL_JSON_OPTIONS = JSONOptions(json_mode=JSONMode.CANONICAL) """:class:`JSONOptions` for Canonical Extended JSON. .. seealso:: The documentation for :const:`bson.json_util.JSONMode.CANONICAL`. .. versionadded:: 3.5 """ RELAXED_JSON_OPTIONS = JSONOptions(json_mode=JSONMode.RELAXED) """:class:`JSONOptions` for Relaxed Extended JSON. .. seealso:: The documentation for :const:`bson.json_util.JSONMode.RELAXED`. .. versionadded:: 3.5 """ STRICT_JSON_OPTIONS = JSONOptions( strict_number_long=True, datetime_representation=DatetimeRepresentation.ISO8601, strict_uuid=True) """**DEPRECATED** - :class:`JSONOptions` for MongoDB Extended JSON's *Strict mode* encoding. .. versionadded:: 3.4 .. versionchanged:: 3.5 Deprecated. Use :const:`RELAXED_JSON_OPTIONS` or :const:`CANONICAL_JSON_OPTIONS` instead. """ def dumps(obj, *args, **kwargs): """Helper function that wraps :func:`json.dumps`. Recursive function that handles all BSON types including :class:`~bson.binary.Binary` and :class:`~bson.code.Code`. :Parameters: - `json_options`: A :class:`JSONOptions` instance used to modify the encoding of MongoDB Extended JSON types. Defaults to :const:`DEFAULT_JSON_OPTIONS`. .. versionchanged:: 3.4 Accepts optional parameter `json_options`. See :class:`JSONOptions`. .. versionchanged:: 2.7 Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef instances. """ json_options = kwargs.pop("json_options", DEFAULT_JSON_OPTIONS) return json.dumps(_json_convert(obj, json_options), *args, **kwargs) def loads(s, *args, **kwargs): """Helper function that wraps :func:`json.loads`. Automatically passes the object_hook for BSON type conversion. Raises ``TypeError``, ``ValueError``, ``KeyError``, or :exc:`~bson.errors.InvalidId` on invalid MongoDB Extended JSON. :Parameters: - `json_options`: A :class:`JSONOptions` instance used to modify the decoding of MongoDB Extended JSON types. Defaults to :const:`DEFAULT_JSON_OPTIONS`. .. versionchanged:: 3.5 Parses Relaxed and Canonical Extended JSON as well as PyMongo's legacy format. Now raises ``TypeError`` or ``ValueError`` when parsing JSON type wrappers with values of the wrong type or any extra keys. .. versionchanged:: 3.4 Accepts optional parameter `json_options`. See :class:`JSONOptions`. """ json_options = kwargs.pop("json_options", DEFAULT_JSON_OPTIONS) if _HAS_OBJECT_PAIRS_HOOK: kwargs["object_pairs_hook"] = lambda pairs: object_pairs_hook( pairs, json_options) else: kwargs["object_hook"] = lambda obj: object_hook(obj, json_options) return json.loads(s, *args, **kwargs) def _json_convert(obj, json_options=DEFAULT_JSON_OPTIONS): """Recursive helper method that converts BSON types so they can be converted into json. """ if hasattr(obj, 'iteritems') or hasattr(obj, 'items'): # PY3 support return SON(((k, _json_convert(v, json_options)) for k, v in iteritems(obj))) elif hasattr(obj, '__iter__') and not isinstance(obj, (text_type, bytes)): return list((_json_convert(v, json_options) for v in obj)) try: return default(obj, json_options) except TypeError: return obj def object_pairs_hook(pairs, json_options=DEFAULT_JSON_OPTIONS): return object_hook(json_options.document_class(pairs), json_options) def object_hook(dct, json_options=DEFAULT_JSON_OPTIONS): if "$oid" in dct: return _parse_canonical_oid(dct) if "$ref" in dct: return _parse_canonical_dbref(dct) if "$date" in dct: return _parse_canonical_datetime(dct, json_options) if "$regex" in dct: return _parse_legacy_regex(dct) if "$minKey" in dct: return _parse_canonical_minkey(dct) if "$maxKey" in dct: return _parse_canonical_maxkey(dct) if "$binary" in dct: if "$type" in dct: return _parse_legacy_binary(dct, json_options) else: return _parse_canonical_binary(dct, json_options) if "$code" in dct: return _parse_canonical_code(dct) if "$uuid" in dct: return _parse_legacy_uuid(dct) if "$undefined" in dct: return None if "$numberLong" in dct: return _parse_canonical_int64(dct) if "$timestamp" in dct: tsp = dct["$timestamp"] return Timestamp(tsp["t"], tsp["i"]) if "$numberDecimal" in dct: return _parse_canonical_decimal128(dct) if "$dbPointer" in dct: return _parse_canonical_dbpointer(dct) if "$regularExpression" in dct: return _parse_canonical_regex(dct) if "$symbol" in dct: return _parse_canonical_symbol(dct) if "$numberInt" in dct: return _parse_canonical_int32(dct) if "$numberDouble" in dct: return _parse_canonical_double(dct) return dct def _parse_legacy_regex(doc): pattern = doc["$regex"] # Check if this is the $regex query operator. if isinstance(pattern, Regex): return doc flags = 0 # PyMongo always adds $options but some other tools may not. for opt in doc.get("$options", ""): flags |= _RE_OPT_TABLE.get(opt, 0) return Regex(pattern, flags) def _parse_legacy_uuid(doc): """Decode a JSON legacy $uuid to Python UUID.""" if len(doc) != 1: raise TypeError('Bad $uuid, extra field(s): %s' % (doc,)) return uuid.UUID(doc["$uuid"]) def _binary_or_uuid(data, subtype, json_options): # special handling for UUID if subtype == OLD_UUID_SUBTYPE: if json_options.uuid_representation == CSHARP_LEGACY: return uuid.UUID(bytes_le=data) if json_options.uuid_representation == JAVA_LEGACY: data = data[7::-1] + data[:7:-1] return uuid.UUID(bytes=data) if subtype == UUID_SUBTYPE: return uuid.UUID(bytes=data) if PY3 and subtype == 0: return data return Binary(data, subtype) def _parse_legacy_binary(doc, json_options): if isinstance(doc["$type"], int): doc["$type"] = "%02x" % doc["$type"] subtype = int(doc["$type"], 16) if subtype >= 0xffffff80: # Handle mongoexport values subtype = int(doc["$type"][6:], 16) data = base64.b64decode(doc["$binary"].encode()) return _binary_or_uuid(data, subtype, json_options) def _parse_canonical_binary(doc, json_options): binary = doc["$binary"] b64 = binary["base64"] subtype = binary["subType"] if not isinstance(b64, string_type): raise TypeError('$binary base64 must be a string: %s' % (doc,)) if not isinstance(subtype, string_type) or len(subtype) > 2: raise TypeError('$binary subType must be a string at most 2 ' 'characters: %s' % (doc,)) if len(binary) != 2: raise TypeError('$binary must include only "base64" and "subType" ' 'components: %s' % (doc,)) data = base64.b64decode(b64.encode()) return _binary_or_uuid(data, int(subtype, 16), json_options) def _parse_canonical_datetime(doc, json_options): """Decode a JSON datetime to python datetime.datetime.""" dtm = doc["$date"] if len(doc) != 1: raise TypeError('Bad $date, extra field(s): %s' % (doc,)) # mongoexport 2.6 and newer if isinstance(dtm, string_type): # Parse offset if dtm[-1] == 'Z': dt = dtm[:-1] offset = 'Z' elif dtm[-3] == ':': # (+|-)HH:MM dt = dtm[:-6] offset = dtm[-6:] elif dtm[-5] in ('+', '-'): # (+|-)HHMM dt = dtm[:-5] offset = dtm[-5:] elif dtm[-3] in ('+', '-'): # (+|-)HH dt = dtm[:-3] offset = dtm[-3:] else: dt = dtm offset = '' # Parse the optional factional seconds portion. dot_index = dt.rfind('.') microsecond = 0 if dot_index != -1: microsecond = int(float(dt[dot_index:]) * 1000000) dt = dt[:dot_index] aware = datetime.datetime.strptime( dt, "%Y-%m-%dT%H:%M:%S").replace(microsecond=microsecond, tzinfo=utc) if offset and offset != 'Z': if len(offset) == 6: hours, minutes = offset[1:].split(':') secs = (int(hours) * 3600 + int(minutes) * 60) elif len(offset) == 5: secs = (int(offset[1:3]) * 3600 + int(offset[3:]) * 60) elif len(offset) == 3: secs = int(offset[1:3]) * 3600 if offset[0] == "-": secs *= -1 aware = aware - datetime.timedelta(seconds=secs) if json_options.tz_aware: if json_options.tzinfo: aware = aware.astimezone(json_options.tzinfo) return aware else: return aware.replace(tzinfo=None) return bson._millis_to_datetime(int(dtm), json_options) def _parse_canonical_oid(doc): """Decode a JSON ObjectId to bson.objectid.ObjectId.""" if len(doc) != 1: raise TypeError('Bad $oid, extra field(s): %s' % (doc,)) return ObjectId(doc['$oid']) def _parse_canonical_symbol(doc): """Decode a JSON symbol to Python string.""" symbol = doc['$symbol'] if len(doc) != 1: raise TypeError('Bad $symbol, extra field(s): %s' % (doc,)) return text_type(symbol) def _parse_canonical_code(doc): """Decode a JSON code to bson.code.Code.""" for key in doc: if key not in ('$code', '$scope'): raise TypeError('Bad $code, extra field(s): %s' % (doc,)) return Code(doc['$code'], scope=doc.get('$scope')) def _parse_canonical_regex(doc): """Decode a JSON regex to bson.regex.Regex.""" regex = doc['$regularExpression'] if len(doc) != 1: raise TypeError('Bad $regularExpression, extra field(s): %s' % (doc,)) if len(regex) != 2: raise TypeError('Bad $regularExpression must include only "pattern"' 'and "options" components: %s' % (doc,)) return Regex(regex['pattern'], regex['options']) def _parse_canonical_dbref(doc): """Decode a JSON DBRef to bson.dbref.DBRef.""" for key in doc: if key.startswith('$') and key not in _DBREF_KEYS: # Other keys start with $, so dct cannot be parsed as a DBRef. return doc return DBRef(doc.pop('$ref'), doc.pop('$id'), database=doc.pop('$db', None), **doc) def _parse_canonical_dbpointer(doc): """Decode a JSON (deprecated) DBPointer to bson.dbref.DBRef.""" dbref = doc['$dbPointer'] if len(doc) != 1: raise TypeError('Bad $dbPointer, extra field(s): %s' % (doc,)) if isinstance(dbref, DBRef): dbref_doc = dbref.as_doc() # DBPointer must not contain $db in its value. if dbref.database is not None: raise TypeError( 'Bad $dbPointer, extra field $db: %s' % (dbref_doc,)) if not isinstance(dbref.id, ObjectId): raise TypeError( 'Bad $dbPointer, $id must be an ObjectId: %s' % (dbref_doc,)) if len(dbref_doc) != 2: raise TypeError( 'Bad $dbPointer, extra field(s) in DBRef: %s' % (dbref_doc,)) return dbref else: raise TypeError('Bad $dbPointer, expected a DBRef: %s' % (doc,)) def _parse_canonical_int32(doc): """Decode a JSON int32 to python int.""" i_str = doc['$numberInt'] if len(doc) != 1: raise TypeError('Bad $numberInt, extra field(s): %s' % (doc,)) if not isinstance(i_str, string_type): raise TypeError('$numberInt must be string: %s' % (doc,)) return int(i_str) def _parse_canonical_int64(doc): """Decode a JSON int64 to bson.int64.Int64.""" l_str = doc['$numberLong'] if len(doc) != 1: raise TypeError('Bad $numberLong, extra field(s): %s' % (doc,)) return Int64(l_str) def _parse_canonical_double(doc): """Decode a JSON double to python float.""" d_str = doc['$numberDouble'] if len(doc) != 1: raise TypeError('Bad $numberDouble, extra field(s): %s' % (doc,)) if not isinstance(d_str, string_type): raise TypeError('$numberDouble must be string: %s' % (doc,)) return float(d_str) def _parse_canonical_decimal128(doc): """Decode a JSON decimal128 to bson.decimal128.Decimal128.""" d_str = doc['$numberDecimal'] if len(doc) != 1: raise TypeError('Bad $numberDecimal, extra field(s): %s' % (doc,)) if not isinstance(d_str, string_type): raise TypeError('$numberDecimal must be string: %s' % (doc,)) return Decimal128(d_str) def _parse_canonical_minkey(doc): """Decode a JSON MinKey to bson.min_key.MinKey.""" if doc['$minKey'] is not 1: raise TypeError('$minKey value must be 1: %s' % (doc,)) if len(doc) != 1: raise TypeError('Bad $minKey, extra field(s): %s' % (doc,)) return MinKey() def _parse_canonical_maxkey(doc): """Decode a JSON MaxKey to bson.max_key.MaxKey.""" if doc['$maxKey'] is not 1: raise TypeError('$maxKey value must be 1: %s', (doc,)) if len(doc) != 1: raise TypeError('Bad $minKey, extra field(s): %s' % (doc,)) return MaxKey() def _encode_binary(data, subtype, json_options): if json_options.json_mode == JSONMode.LEGACY: return SON([ ('$binary', base64.b64encode(data).decode()), ('$type', "%02x" % subtype)]) return {'$binary': SON([ ('base64', base64.b64encode(data).decode()), ('subType', "%02x" % subtype)])} def default(obj, json_options=DEFAULT_JSON_OPTIONS): # We preserve key order when rendering SON, DBRef, etc. as JSON by # returning a SON for those types instead of a dict. if isinstance(obj, ObjectId): return {"$oid": str(obj)} if isinstance(obj, DBRef): return _json_convert(obj.as_doc(), json_options=json_options) if isinstance(obj, datetime.datetime): if (json_options.datetime_representation == DatetimeRepresentation.ISO8601): if not obj.tzinfo: obj = obj.replace(tzinfo=utc) if obj >= EPOCH_AWARE: off = obj.tzinfo.utcoffset(obj) if (off.days, off.seconds, off.microseconds) == (0, 0, 0): tz_string = 'Z' else: tz_string = obj.strftime('%z') millis = int(obj.microsecond / 1000) fracsecs = ".%03d" % (millis,) if millis else "" return {"$date": "%s%s%s" % ( obj.strftime("%Y-%m-%dT%H:%M:%S"), fracsecs, tz_string)} millis = bson._datetime_to_millis(obj) if (json_options.datetime_representation == DatetimeRepresentation.LEGACY): return {"$date": millis} return {"$date": {"$numberLong": str(millis)}} if json_options.strict_number_long and isinstance(obj, Int64): return {"$numberLong": str(obj)} if isinstance(obj, (RE_TYPE, Regex)): flags = "" if obj.flags & re.IGNORECASE: flags += "i" if obj.flags & re.LOCALE: flags += "l" if obj.flags & re.MULTILINE: flags += "m" if obj.flags & re.DOTALL: flags += "s" if obj.flags & re.UNICODE: flags += "u" if obj.flags & re.VERBOSE: flags += "x" if isinstance(obj.pattern, text_type): pattern = obj.pattern else: pattern = obj.pattern.decode('utf-8') if json_options.json_mode == JSONMode.LEGACY: return SON([("$regex", pattern), ("$options", flags)]) return {'$regularExpression': SON([("pattern", pattern), ("options", flags)])} if isinstance(obj, MinKey): return {"$minKey": 1} if isinstance(obj, MaxKey): return {"$maxKey": 1} if isinstance(obj, Timestamp): return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])} if isinstance(obj, Code): if obj.scope is None: return {'$code': str(obj)} return SON([ ('$code', str(obj)), ('$scope', _json_convert(obj.scope, json_options))]) if isinstance(obj, Binary): return _encode_binary(obj, obj.subtype, json_options) if PY3 and isinstance(obj, bytes): return _encode_binary(obj, 0, json_options) if isinstance(obj, uuid.UUID): if json_options.strict_uuid: data = obj.bytes subtype = OLD_UUID_SUBTYPE if json_options.uuid_representation == CSHARP_LEGACY: data = obj.bytes_le elif json_options.uuid_representation == JAVA_LEGACY: data = data[7::-1] + data[:7:-1] elif json_options.uuid_representation == UUID_SUBTYPE: subtype = UUID_SUBTYPE return _encode_binary(data, subtype, json_options) else: return {"$uuid": obj.hex} if isinstance(obj, Decimal128): return {"$numberDecimal": str(obj)} if isinstance(obj, bool): return obj if (json_options.json_mode == JSONMode.CANONICAL and isinstance(obj, integer_types)): if -2 ** 31 <= obj < 2 ** 31: return {'$numberInt': text_type(obj)} return {'$numberLong': text_type(obj)} if json_options.json_mode != JSONMode.LEGACY and isinstance(obj, float): if math.isnan(obj): return {'$numberDouble': 'NaN'} elif math.isinf(obj): representation = 'Infinity' if obj > 0 else '-Infinity' return {'$numberDouble': representation} elif json_options.json_mode == JSONMode.CANONICAL: # repr() will return the shortest string guaranteed to produce the # original value, when float() is called on it. str produces a # shorter string in Python 2. return {'$numberDouble': text_type(repr(obj))} raise TypeError("%r is not JSON serializable" % obj) pymongo-3.6.1/bson/encoding_helpers.h0000644000076600000240000000154313156613521020063 0ustar shanestaff00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef ENCODING_HELPERS_H #define ENCODING_HELPERS_H typedef enum { VALID, NOT_UTF_8, HAS_NULL } result_t; result_t check_string(const unsigned char* string, const int length, const char check_utf8, const char check_null); #endif pymongo-3.6.1/bson/buffer.h0000644000076600000240000000401413156613521016020 0ustar shanestaff00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef BUFFER_H #define BUFFER_H /* Note: if any of these functions return a failure condition then the buffer * has already been freed. */ /* A buffer */ typedef struct buffer* buffer_t; /* A position in the buffer */ typedef int buffer_position; /* Allocate and return a new buffer. * Return NULL on allocation failure. */ buffer_t buffer_new(void); /* Free the memory allocated for `buffer`. * Return non-zero on failure. */ int buffer_free(buffer_t buffer); /* Save `size` bytes from the current position in `buffer` (and grow if needed). * Return offset for writing, or -1 on allocation failure. */ buffer_position buffer_save_space(buffer_t buffer, int size); /* Write `size` bytes from `data` to `buffer` (and grow if needed). * Return non-zero on allocation failure. */ int buffer_write(buffer_t buffer, const char* data, int size); /* Write `size` bytes from `data` to `buffer` at position `position`. * Does not change the internal position of `buffer`. * Return non-zero if buffer isn't large enough for write. */ int buffer_write_at_position(buffer_t buffer, buffer_position position, const char* data, int size); /* Getters for the internals of a buffer_t. * Should try to avoid using these as much as possible * since they break the abstraction. */ buffer_position buffer_get_position(buffer_t buffer); char* buffer_get_buffer(buffer_t buffer); void buffer_update_position(buffer_t buffer, buffer_position new_position); #endif pymongo-3.6.1/bson/time64_config.h0000644000076600000240000000322213156613521017204 0ustar shanestaff00000000000000/* Configuration ------------- Define as appropriate for your system. Sensible defaults provided. */ #ifndef TIME64_CONFIG_H # define TIME64_CONFIG_H /* Debugging TIME_64_DEBUG Define if you want debugging messages */ /* #define TIME_64_DEBUG */ /* INT_64_T A 64 bit integer type to use to store time and others. Must be defined. */ #define INT_64_T long long /* USE_TM64 Should we use a 64 bit safe replacement for tm? This will let you go past year 2 billion but the struct will be incompatible with tm. Conversion functions will be provided. */ /* #define USE_TM64 */ /* Availability of system functions. HAS_GMTIME_R Define if your system has gmtime_r() HAS_LOCALTIME_R Define if your system has localtime_r() HAS_TIMEGM Define if your system has timegm(), a GNU extension. */ #if !defined(WIN32) && !defined(_MSC_VER) #define HAS_GMTIME_R #define HAS_LOCALTIME_R #endif /* #define HAS_TIMEGM */ /* Details of non-standard tm struct elements. HAS_TM_TM_GMTOFF True if your tm struct has a "tm_gmtoff" element. A BSD extension. HAS_TM_TM_ZONE True if your tm struct has a "tm_zone" element. A BSD extension. */ /* #define HAS_TM_TM_GMTOFF */ /* #define HAS_TM_TM_ZONE */ /* USE_SYSTEM_LOCALTIME USE_SYSTEM_GMTIME USE_SYSTEM_MKTIME USE_SYSTEM_TIMEGM Should we use the system functions if the time is inside their range? Your system localtime() is probably more accurate, but our gmtime() is fast and safe. */ #define USE_SYSTEM_LOCALTIME /* #define USE_SYSTEM_GMTIME */ #define USE_SYSTEM_MKTIME /* #define USE_SYSTEM_TIMEGM */ #endif /* TIME64_CONFIG_H */ pymongo-3.6.1/bson/time64_limits.h0000644000076600000240000000272413245621354017250 0ustar shanestaff00000000000000/* Maximum and minimum inputs your system's respective time functions can correctly handle. time64.h will use your system functions if the input falls inside these ranges and corresponding USE_SYSTEM_* constant is defined. */ #ifndef TIME64_LIMITS_H #define TIME64_LIMITS_H /* Max/min for localtime() */ #define SYSTEM_LOCALTIME_MAX 2147483647 #define SYSTEM_LOCALTIME_MIN -2147483647-1 /* Max/min for gmtime() */ #define SYSTEM_GMTIME_MAX 2147483647 #define SYSTEM_GMTIME_MIN -2147483647-1 /* Max/min for mktime() */ static const struct tm SYSTEM_MKTIME_MAX = { 7, 14, 19, 18, 0, 138, 1, 17, 0 #ifdef HAS_TM_TM_GMTOFF ,-28800 #endif #ifdef HAS_TM_TM_ZONE ,"PST" #endif }; static const struct tm SYSTEM_MKTIME_MIN = { 52, 45, 12, 13, 11, 1, 5, 346, 0 #ifdef HAS_TM_TM_GMTOFF ,-28800 #endif #ifdef HAS_TM_TM_ZONE ,"PST" #endif }; /* Max/min for timegm() */ #ifdef HAS_TIMEGM static const struct tm SYSTEM_TIMEGM_MAX = { 7, 14, 3, 19, 0, 138, 2, 18, 0 #ifdef HAS_TM_TM_GMTOFF ,0 #endif #ifdef HAS_TM_TM_ZONE ,"UTC" #endif }; static const struct tm SYSTEM_TIMEGM_MIN = { 52, 45, 20, 13, 11, 1, 5, 346, 0 #ifdef HAS_TM_TM_GMTOFF ,0 #endif #ifdef HAS_TM_TM_ZONE ,"UTC" #endif }; #endif /* HAS_TIMEGM */ #endif /* TIME64_LIMITS_H */ pymongo-3.6.1/bson/_cbsonmodule.c0000644000076600000240000027640513245621354017234 0ustar shanestaff00000000000000/* * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains C implementations of some of the functions * needed by the bson module. If possible, these implementations * should be used to speed up BSON encoding and decoding. */ #include "Python.h" #include "datetime.h" #include "buffer.h" #include "time64.h" #include "encoding_helpers.h" #define _CBSON_MODULE #include "_cbsonmodule.h" /* New module state and initialization code. * See the module-initialization-and-state * section in the following doc: * http://docs.python.org/release/3.1.3/howto/cporting.html * which references the following pep: * http://www.python.org/dev/peps/pep-3121/ * */ struct module_state { PyObject* Binary; PyObject* Code; PyObject* ObjectId; PyObject* DBRef; PyObject* Regex; PyObject* UUID; PyObject* Timestamp; PyObject* MinKey; PyObject* MaxKey; PyObject* UTC; PyTypeObject* REType; PyObject* BSONInt64; PyObject* Decimal128; PyObject* Mapping; PyObject* CodecOptions; }; /* The Py_TYPE macro was introduced in CPython 2.6 */ #ifndef Py_TYPE #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #endif #if PY_MAJOR_VERSION >= 3 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) #else #define GETSTATE(m) (&_state) static struct module_state _state; #endif /* Maximum number of regex flags */ #define FLAGS_SIZE 7 /* Default UUID representation type code. */ #define PYTHON_LEGACY 3 /* Other UUID representations. */ #define STANDARD 4 #define JAVA_LEGACY 5 #define CSHARP_LEGACY 6 #define BSON_MAX_SIZE 2147483647 /* The smallest possible BSON document, i.e. "{}" */ #define BSON_MIN_SIZE 5 /* Get an error class from the bson.errors module. * * Returns a new ref */ static PyObject* _error(char* name) { PyObject* error; PyObject* errors = PyImport_ImportModule("bson.errors"); if (!errors) { return NULL; } error = PyObject_GetAttrString(errors, name); Py_DECREF(errors); return error; } /* Safely downcast from Py_ssize_t to int, setting an * exception and returning -1 on error. */ static int _downcast_and_check(Py_ssize_t size, uint8_t extra) { if (size > BSON_MAX_SIZE || ((BSON_MAX_SIZE - extra) < size)) { PyObject* InvalidStringData = _error("InvalidStringData"); if (InvalidStringData) { PyErr_SetString(InvalidStringData, "String length must be <= 2147483647"); Py_DECREF(InvalidStringData); } return -1; } return (int)size + extra; } static PyObject* elements_to_dict(PyObject* self, const char* string, unsigned max, const codec_options_t* options); static int _write_element_to_buffer(PyObject* self, buffer_t buffer, int type_byte, PyObject* value, unsigned char check_keys, const codec_options_t* options); /* Date stuff */ static PyObject* datetime_from_millis(long long millis) { /* To encode a datetime instance like datetime(9999, 12, 31, 23, 59, 59, 999999) * we follow these steps: * 1. Calculate a timestamp in seconds: 253402300799 * 2. Multiply that by 1000: 253402300799000 * 3. Add in microseconds divided by 1000 253402300799999 * * (Note: BSON doesn't support microsecond accuracy, hence the rounding.) * * To decode we could do: * 1. Get seconds: timestamp / 1000: 253402300799 * 2. Get micros: (timestamp % 1000) * 1000: 999000 * Resulting in datetime(9999, 12, 31, 23, 59, 59, 999000) -- the expected result * * Now what if the we encode (1, 1, 1, 1, 1, 1, 111111)? * 1. and 2. gives: -62135593139000 * 3. Gives us: -62135593138889 * * Now decode: * 1. Gives us: -62135593138 * 2. Gives us: -889000 * Resulting in datetime(1, 1, 1, 1, 1, 2, 15888216) -- an invalid result * * If instead to decode we do: * diff = ((millis % 1000) + 1000) % 1000: 111 * seconds = (millis - diff) / 1000: -62135593139 * micros = diff * 1000 111000 * Resulting in datetime(1, 1, 1, 1, 1, 1, 111000) -- the expected result */ int diff = (int)(((millis % 1000) + 1000) % 1000); int microseconds = diff * 1000; Time64_T seconds = (millis - diff) / 1000; struct TM timeinfo; gmtime64_r(&seconds, &timeinfo); return PyDateTime_FromDateAndTime(timeinfo.tm_year + 1900, timeinfo.tm_mon + 1, timeinfo.tm_mday, timeinfo.tm_hour, timeinfo.tm_min, timeinfo.tm_sec, microseconds); } static long long millis_from_datetime(PyObject* datetime) { struct TM timeinfo; long long millis; timeinfo.tm_year = PyDateTime_GET_YEAR(datetime) - 1900; timeinfo.tm_mon = PyDateTime_GET_MONTH(datetime) - 1; timeinfo.tm_mday = PyDateTime_GET_DAY(datetime); timeinfo.tm_hour = PyDateTime_DATE_GET_HOUR(datetime); timeinfo.tm_min = PyDateTime_DATE_GET_MINUTE(datetime); timeinfo.tm_sec = PyDateTime_DATE_GET_SECOND(datetime); millis = timegm64(&timeinfo) * 1000; millis += PyDateTime_DATE_GET_MICROSECOND(datetime) / 1000; return millis; } /* Just make this compatible w/ the old API. */ int buffer_write_bytes(buffer_t buffer, const char* data, int size) { if (buffer_write(buffer, data, size)) { PyErr_NoMemory(); return 0; } return 1; } int buffer_write_double(buffer_t buffer, double data) { double data_le = BSON_DOUBLE_TO_LE(data); return buffer_write_bytes(buffer, (const char*)&data_le, 8); } int buffer_write_int32(buffer_t buffer, int32_t data) { uint32_t data_le = BSON_UINT32_TO_LE(data); return buffer_write_bytes(buffer, (const char*)&data_le, 4); } int buffer_write_int64(buffer_t buffer, int64_t data) { uint64_t data_le = BSON_UINT64_TO_LE(data); return buffer_write_bytes(buffer, (const char*)&data_le, 8); } void buffer_write_int32_at_position(buffer_t buffer, int position, int32_t data) { uint32_t data_le = BSON_UINT32_TO_LE(data); memcpy(buffer_get_buffer(buffer) + position, &data_le, 4); } static int write_unicode(buffer_t buffer, PyObject* py_string) { int size; const char* data; PyObject* encoded = PyUnicode_AsUTF8String(py_string); if (!encoded) { return 0; } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AS_STRING(encoded); #else data = PyString_AS_STRING(encoded); #endif if (!data) goto unicodefail; #if PY_MAJOR_VERSION >= 3 if ((size = _downcast_and_check(PyBytes_GET_SIZE(encoded), 1)) == -1) #else if ((size = _downcast_and_check(PyString_GET_SIZE(encoded), 1)) == -1) #endif goto unicodefail; if (!buffer_write_int32(buffer, (int32_t)size)) goto unicodefail; if (!buffer_write_bytes(buffer, data, size)) goto unicodefail; Py_DECREF(encoded); return 1; unicodefail: Py_DECREF(encoded); return 0; } /* returns 0 on failure */ static int write_string(buffer_t buffer, PyObject* py_string) { int size; const char* data; #if PY_MAJOR_VERSION >= 3 if (PyUnicode_Check(py_string)){ return write_unicode(buffer, py_string); } data = PyBytes_AsString(py_string); #else data = PyString_AsString(py_string); #endif if (!data) { return 0; } #if PY_MAJOR_VERSION >= 3 if ((size = _downcast_and_check(PyBytes_Size(py_string), 1)) == -1) #else if ((size = _downcast_and_check(PyString_Size(py_string), 1)) == -1) #endif return 0; if (!buffer_write_int32(buffer, (int32_t)size)) { return 0; } if (!buffer_write_bytes(buffer, data, size)) { return 0; } return 1; } /* * Are we in the main interpreter or a sub-interpreter? * Useful for deciding if we can use cached pure python * types in mod_wsgi. */ static int _in_main_interpreter(void) { static PyInterpreterState* main_interpreter = NULL; PyInterpreterState* interpreter; if (main_interpreter == NULL) { interpreter = PyInterpreterState_Head(); while (PyInterpreterState_Next(interpreter)) interpreter = PyInterpreterState_Next(interpreter); main_interpreter = interpreter; } return (main_interpreter == PyThreadState_Get()->interp); } /* * Get a reference to a pure python type. If we are in the * main interpreter return the cached object, otherwise import * the object we need and return it instead. */ static PyObject* _get_object(PyObject* object, char* module_name, char* object_name) { if (_in_main_interpreter()) { Py_XINCREF(object); return object; } else { PyObject* imported = NULL; PyObject* module = PyImport_ImportModule(module_name); if (!module) return NULL; imported = PyObject_GetAttrString(module, object_name); Py_DECREF(module); return imported; } } /* Load a Python object to cache. * * Returns non-zero on failure. */ static int _load_object(PyObject** object, char* module_name, char* object_name) { PyObject* module; module = PyImport_ImportModule(module_name); if (!module) { return 1; } *object = PyObject_GetAttrString(module, object_name); Py_DECREF(module); return (*object) ? 0 : 2; } /* Load all Python objects to cache. * * Returns non-zero on failure. */ static int _load_python_objects(PyObject* module) { PyObject* empty_string; PyObject* re_compile; PyObject* compiled; struct module_state *state = GETSTATE(module); if (_load_object(&state->Binary, "bson.binary", "Binary") || _load_object(&state->Code, "bson.code", "Code") || _load_object(&state->ObjectId, "bson.objectid", "ObjectId") || _load_object(&state->DBRef, "bson.dbref", "DBRef") || _load_object(&state->Timestamp, "bson.timestamp", "Timestamp") || _load_object(&state->MinKey, "bson.min_key", "MinKey") || _load_object(&state->MaxKey, "bson.max_key", "MaxKey") || _load_object(&state->UTC, "bson.tz_util", "utc") || _load_object(&state->Regex, "bson.regex", "Regex") || _load_object(&state->BSONInt64, "bson.int64", "Int64") || _load_object(&state->Decimal128, "bson.decimal128", "Decimal128") || _load_object(&state->UUID, "uuid", "UUID") || #if PY_MAJOR_VERSION >= 3 _load_object(&state->Mapping, "collections.abc", "Mapping") || #else _load_object(&state->Mapping, "collections", "Mapping") || #endif _load_object(&state->CodecOptions, "bson.codec_options", "CodecOptions")) { return 1; } /* Reload our REType hack too. */ #if PY_MAJOR_VERSION >= 3 empty_string = PyBytes_FromString(""); #else empty_string = PyString_FromString(""); #endif if (empty_string == NULL) { state->REType = NULL; return 1; } if (_load_object(&re_compile, "re", "compile")) { state->REType = NULL; Py_DECREF(empty_string); return 1; } compiled = PyObject_CallFunction(re_compile, "O", empty_string); Py_DECREF(re_compile); if (compiled == NULL) { state->REType = NULL; Py_DECREF(empty_string); return 1; } Py_INCREF(Py_TYPE(compiled)); state->REType = Py_TYPE(compiled); Py_DECREF(empty_string); Py_DECREF(compiled); return 0; } /* * Get the _type_marker from an Object. * * Return the type marker, 0 if there is no marker, or -1 on failure. */ static long _type_marker(PyObject* object) { PyObject* type_marker = NULL; long type = 0; if (PyObject_HasAttrString(object, "_type_marker")) { type_marker = PyObject_GetAttrString(object, "_type_marker"); if (type_marker == NULL) { return -1; } } /* * Python objects with broken __getattr__ implementations could return * arbitrary types for a call to PyObject_GetAttrString. For example * pymongo.database.Database returns a new Collection instance for * __getattr__ calls with names that don't match an existing attribute * or method. In some cases "value" could be a subtype of something * we know how to serialize. Make a best effort to encode these types. */ #if PY_MAJOR_VERSION >= 3 if (type_marker && PyLong_CheckExact(type_marker)) { type = PyLong_AsLong(type_marker); #else if (type_marker && PyInt_CheckExact(type_marker)) { type = PyInt_AsLong(type_marker); #endif Py_DECREF(type_marker); /* * Py(Long|Int)_AsLong returns -1 for error but -1 is a valid value * so we call PyErr_Occurred to differentiate. */ if (type == -1 && PyErr_Occurred()) { return -1; } } else { Py_XDECREF(type_marker); } return type; } /* Fill out a codec_options_t* from a CodecOptions object. Use with the "O&" * format spec in PyArg_ParseTuple. * * Return 1 on success. options->document_class is a new reference. * Return 0 on failure. */ int convert_codec_options(PyObject* options_obj, void* p) { codec_options_t* options = (codec_options_t*)p; long type_marker; options->unicode_decode_error_handler = NULL; if (!PyArg_ParseTuple(options_obj, "ObbzO", &options->document_class, &options->tz_aware, &options->uuid_rep, &options->unicode_decode_error_handler, &options->tzinfo)) { return 0; } type_marker = _type_marker(options->document_class); if (type_marker < 0) return 0; Py_INCREF(options->document_class); Py_INCREF(options->tzinfo); options->options_obj = options_obj; Py_INCREF(options->options_obj); options->is_raw_bson = (101 == type_marker); return 1; } /* Fill out a codec_options_t* with default options. * * Return 1 on success. * Return 0 on failure. */ int default_codec_options(struct module_state* state, codec_options_t* options) { PyObject* options_obj = NULL; PyObject* codec_options_func = _get_object( state->CodecOptions, "bson.codec_options", "CodecOptions"); if (codec_options_func == NULL) { return 0; } options_obj = PyObject_CallFunctionObjArgs(codec_options_func, NULL); Py_DECREF(codec_options_func); if (options_obj == NULL) { return 0; } return convert_codec_options(options_obj, options); } void destroy_codec_options(codec_options_t* options) { Py_CLEAR(options->document_class); Py_CLEAR(options->tzinfo); Py_CLEAR(options->options_obj); } static int write_element_to_buffer(PyObject* self, buffer_t buffer, int type_byte, PyObject* value, unsigned char check_keys, const codec_options_t* options) { int result; if(Py_EnterRecursiveCall(" while encoding an object to BSON ")) return 0; result = _write_element_to_buffer(self, buffer, type_byte, value, check_keys, options); Py_LeaveRecursiveCall(); return result; } static void _fix_java(const char* in, char* out) { int i, j; for (i = 0, j = 7; i < j; i++, j--) { out[i] = in[j]; out[j] = in[i]; } for (i = 8, j = 15; i < j; i++, j--) { out[i] = in[j]; out[j] = in[i]; } } static void _set_cannot_encode(PyObject* value) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyObject* repr = PyObject_Repr(value); if (repr) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromString("Cannot encode object: "); #else PyObject* errmsg = PyString_FromString("Cannot encode object: "); #endif if (errmsg) { #if PY_MAJOR_VERSION >= 3 PyObject* error = PyUnicode_Concat(errmsg, repr); if (error) { PyErr_SetObject(InvalidDocument, error); Py_DECREF(error); } Py_DECREF(errmsg); Py_DECREF(repr); #else PyString_ConcatAndDel(&errmsg, repr); if (errmsg) { PyErr_SetObject(InvalidDocument, errmsg); Py_DECREF(errmsg); } #endif } else { Py_DECREF(repr); } } Py_DECREF(InvalidDocument); } } /* * Encode a builtin Python regular expression or our custom Regex class. * * Sets exception and returns 0 on failure. */ static int _write_regex_to_buffer( buffer_t buffer, int type_byte, PyObject* value) { PyObject* py_flags; PyObject* py_pattern; PyObject* encoded_pattern; long int_flags; char flags[FLAGS_SIZE]; char check_utf8 = 0; const char* pattern_data; int pattern_length, flags_length; result_t status; /* * Both the builtin re type and our Regex class have attributes * "flags" and "pattern". */ py_flags = PyObject_GetAttrString(value, "flags"); if (!py_flags) { return 0; } #if PY_MAJOR_VERSION >= 3 int_flags = PyLong_AsLong(py_flags); #else int_flags = PyInt_AsLong(py_flags); #endif Py_DECREF(py_flags); if (int_flags == -1 && PyErr_Occurred()) { return 0; } py_pattern = PyObject_GetAttrString(value, "pattern"); if (!py_pattern) { return 0; } if (PyUnicode_Check(py_pattern)) { encoded_pattern = PyUnicode_AsUTF8String(py_pattern); Py_DECREF(py_pattern); if (!encoded_pattern) { return 0; } } else { encoded_pattern = py_pattern; check_utf8 = 1; } #if PY_MAJOR_VERSION >= 3 if (!(pattern_data = PyBytes_AsString(encoded_pattern))) { Py_DECREF(encoded_pattern); return 0; } if ((pattern_length = _downcast_and_check(PyBytes_Size(encoded_pattern), 0)) == -1) { Py_DECREF(encoded_pattern); return 0; } #else if (!(pattern_data = PyString_AsString(encoded_pattern))) { Py_DECREF(encoded_pattern); return 0; } if ((pattern_length = _downcast_and_check(PyString_Size(encoded_pattern), 0)) == -1) { Py_DECREF(encoded_pattern); return 0; } #endif status = check_string((const unsigned char*)pattern_data, pattern_length, check_utf8, 1); if (status == NOT_UTF_8) { PyObject* InvalidStringData = _error("InvalidStringData"); if (InvalidStringData) { PyErr_SetString(InvalidStringData, "regex patterns must be valid UTF-8"); Py_DECREF(InvalidStringData); } Py_DECREF(encoded_pattern); return 0; } else if (status == HAS_NULL) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyErr_SetString(InvalidDocument, "regex patterns must not contain the NULL byte"); Py_DECREF(InvalidDocument); } Py_DECREF(encoded_pattern); return 0; } if (!buffer_write_bytes(buffer, pattern_data, pattern_length + 1)) { Py_DECREF(encoded_pattern); return 0; } Py_DECREF(encoded_pattern); flags[0] = 0; if (int_flags & 2) { STRCAT(flags, FLAGS_SIZE, "i"); } if (int_flags & 4) { STRCAT(flags, FLAGS_SIZE, "l"); } if (int_flags & 8) { STRCAT(flags, FLAGS_SIZE, "m"); } if (int_flags & 16) { STRCAT(flags, FLAGS_SIZE, "s"); } if (int_flags & 32) { STRCAT(flags, FLAGS_SIZE, "u"); } if (int_flags & 64) { STRCAT(flags, FLAGS_SIZE, "x"); } flags_length = (int)strlen(flags) + 1; if (!buffer_write_bytes(buffer, flags, flags_length)) { return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x0B; return 1; } /* Write a single value to the buffer (also write its type_byte, for which * space has already been reserved. * * returns 0 on failure */ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, int type_byte, PyObject* value, unsigned char check_keys, const codec_options_t* options) { struct module_state *state = GETSTATE(self); PyObject* mapping_type; PyObject* uuid_type; /* * Don't use PyObject_IsInstance for our custom types. It causes * problems with python sub interpreters. Our custom types should * have a _type_marker attribute, which we can switch on instead. */ long type = _type_marker(value); if (type < 0) { return 0; } switch (type) { case 5: { /* Binary */ PyObject* subtype_object; char subtype; const char* data; int size; *(buffer_get_buffer(buffer) + type_byte) = 0x05; subtype_object = PyObject_GetAttrString(value, "subtype"); if (!subtype_object) { return 0; } #if PY_MAJOR_VERSION >= 3 subtype = (char)PyLong_AsLong(subtype_object); #else subtype = (char)PyInt_AsLong(subtype_object); #endif if (subtype == -1) { Py_DECREF(subtype_object); return 0; } #if PY_MAJOR_VERSION >= 3 size = _downcast_and_check(PyBytes_Size(value), 0); #else size = _downcast_and_check(PyString_Size(value), 0); #endif if (size == -1) { Py_DECREF(subtype_object); return 0; } Py_DECREF(subtype_object); if (subtype == 2) { #if PY_MAJOR_VERSION >= 3 int other_size = _downcast_and_check(PyBytes_Size(value), 4); #else int other_size = _downcast_and_check(PyString_Size(value), 4); #endif if (other_size == -1) return 0; if (!buffer_write_int32(buffer, other_size)) { return 0; } if (!buffer_write_bytes(buffer, &subtype, 1)) { return 0; } } if (!buffer_write_int32(buffer, size)) { return 0; } if (subtype != 2) { if (!buffer_write_bytes(buffer, &subtype, 1)) { return 0; } } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AsString(value); #else data = PyString_AsString(value); #endif if (!data) { return 0; } if (!buffer_write_bytes(buffer, data, size)) { return 0; } return 1; } case 7: { /* ObjectId */ const char* data; PyObject* pystring = PyObject_GetAttrString(value, "_ObjectId__id"); if (!pystring) { return 0; } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AsString(pystring); #else data = PyString_AsString(pystring); #endif if (!data) { Py_DECREF(pystring); return 0; } if (!buffer_write_bytes(buffer, data, 12)) { Py_DECREF(pystring); return 0; } Py_DECREF(pystring); *(buffer_get_buffer(buffer) + type_byte) = 0x07; return 1; } case 11: { /* Regex */ return _write_regex_to_buffer(buffer, type_byte, value); } case 13: { /* Code */ int start_position, length_location, length; PyObject* scope = PyObject_GetAttrString(value, "scope"); if (!scope) { return 0; } if (scope == Py_None) { Py_DECREF(scope); *(buffer_get_buffer(buffer) + type_byte) = 0x0D; return write_string(buffer, value); } *(buffer_get_buffer(buffer) + type_byte) = 0x0F; start_position = buffer_get_position(buffer); /* save space for length */ length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyErr_NoMemory(); Py_DECREF(scope); return 0; } if (!write_string(buffer, value)) { Py_DECREF(scope); return 0; } if (!write_dict(self, buffer, scope, 0, options, 0)) { Py_DECREF(scope); return 0; } Py_DECREF(scope); length = buffer_get_position(buffer) - start_position; buffer_write_int32_at_position( buffer, length_location, (int32_t)length); return 1; } case 17: { /* Timestamp */ PyObject* obj; unsigned long i; obj = PyObject_GetAttrString(value, "inc"); if (!obj) { return 0; } i = PyLong_AsUnsignedLong(obj); Py_DECREF(obj); if (i == (unsigned long)-1 && PyErr_Occurred()) { return 0; } if (!buffer_write_int32(buffer, (int32_t)i)) { return 0; } obj = PyObject_GetAttrString(value, "time"); if (!obj) { return 0; } i = PyLong_AsUnsignedLong(obj); Py_DECREF(obj); if (i == (unsigned long)-1 && PyErr_Occurred()) { return 0; } if (!buffer_write_int32(buffer, (int32_t)i)) { return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x11; return 1; } case 18: { /* Int64 */ const long long ll = PyLong_AsLongLong(value); if (PyErr_Occurred()) { /* Overflow */ PyErr_SetString(PyExc_OverflowError, "MongoDB can only handle up to 8-byte ints"); return 0; } if (!buffer_write_int64(buffer, (int64_t)ll)) { return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x12; return 1; } case 19: { /* Decimal128 */ const char* data; PyObject* pystring = PyObject_GetAttrString(value, "bid"); if (!pystring) { return 0; } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AsString(pystring); #else data = PyString_AsString(pystring); #endif if (!data) { Py_DECREF(pystring); return 0; } if (!buffer_write_bytes(buffer, data, 16)) { Py_DECREF(pystring); return 0; } Py_DECREF(pystring); *(buffer_get_buffer(buffer) + type_byte) = 0x13; return 1; } case 100: { /* DBRef */ PyObject* as_doc = PyObject_CallMethod(value, "as_doc", NULL); if (!as_doc) { return 0; } if (!write_dict(self, buffer, as_doc, 0, options, 0)) { Py_DECREF(as_doc); return 0; } Py_DECREF(as_doc); *(buffer_get_buffer(buffer) + type_byte) = 0x03; return 1; } case 101: { /* RawBSONDocument */ char* raw_bson_document_bytes; Py_ssize_t raw_bson_document_bytes_len; int raw_bson_document_bytes_len_int; PyObject* raw_bson_document_bytes_obj = PyObject_GetAttrString(value, "raw"); if (!raw_bson_document_bytes_obj) { return 0; } #if PY_MAJOR_VERSION >= 3 if (-1 == PyBytes_AsStringAndSize(raw_bson_document_bytes_obj, &raw_bson_document_bytes, &raw_bson_document_bytes_len)) { #else if (-1 == PyString_AsStringAndSize(raw_bson_document_bytes_obj, &raw_bson_document_bytes, &raw_bson_document_bytes_len)) { #endif Py_DECREF(raw_bson_document_bytes_obj); return 0; } raw_bson_document_bytes_len_int = _downcast_and_check( raw_bson_document_bytes_len, 0); if (-1 == raw_bson_document_bytes_len_int) { Py_DECREF(raw_bson_document_bytes_obj); return 0; } if(!buffer_write_bytes(buffer, raw_bson_document_bytes, raw_bson_document_bytes_len_int)) { Py_DECREF(raw_bson_document_bytes_obj); return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x03; Py_DECREF(raw_bson_document_bytes_obj); return 1; } case 255: { /* MinKey */ *(buffer_get_buffer(buffer) + type_byte) = 0xFF; return 1; } case 127: { /* MaxKey */ *(buffer_get_buffer(buffer) + type_byte) = 0x7F; return 1; } } /* No _type_marker attibute or not one of our types. */ if (PyBool_Check(value)) { const char c = (value == Py_True) ? 0x01 : 0x00; *(buffer_get_buffer(buffer) + type_byte) = 0x08; return buffer_write_bytes(buffer, &c, 1); } #if PY_MAJOR_VERSION >= 3 else if (PyLong_Check(value)) { const long long_value = PyLong_AsLong(value); #else else if (PyInt_Check(value)) { const long long_value = PyInt_AsLong(value); #endif const int int_value = (int)long_value; if (PyErr_Occurred() || long_value != int_value) { /* Overflow */ long long long_long_value; PyErr_Clear(); long_long_value = PyLong_AsLongLong(value); if (PyErr_Occurred()) { /* Overflow AGAIN */ PyErr_SetString(PyExc_OverflowError, "MongoDB can only handle up to 8-byte ints"); return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x12; return buffer_write_int64(buffer, (int64_t)long_long_value); } *(buffer_get_buffer(buffer) + type_byte) = 0x10; return buffer_write_int32(buffer, (int32_t)int_value); #if PY_MAJOR_VERSION < 3 } else if (PyLong_Check(value)) { const long long long_long_value = PyLong_AsLongLong(value); if (PyErr_Occurred()) { /* Overflow */ PyErr_SetString(PyExc_OverflowError, "MongoDB can only handle up to 8-byte ints"); return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x12; return buffer_write_int64(buffer, (int64_t)long_long_value); #endif } else if (PyFloat_Check(value)) { const double d = PyFloat_AsDouble(value); *(buffer_get_buffer(buffer) + type_byte) = 0x01; return buffer_write_double(buffer, d); } else if (value == Py_None) { *(buffer_get_buffer(buffer) + type_byte) = 0x0A; return 1; } else if (PyDict_Check(value)) { *(buffer_get_buffer(buffer) + type_byte) = 0x03; return write_dict(self, buffer, value, check_keys, options, 0); } else if (PyList_Check(value) || PyTuple_Check(value)) { Py_ssize_t items, i; int start_position, length_location, length; char zero = 0; *(buffer_get_buffer(buffer) + type_byte) = 0x04; start_position = buffer_get_position(buffer); /* save space for length */ length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyErr_NoMemory(); return 0; } if ((items = PySequence_Size(value)) > BSON_MAX_SIZE) { PyObject* BSONError = _error("BSONError"); if (BSONError) { PyErr_SetString(BSONError, "Too many items to serialize."); Py_DECREF(BSONError); } return 0; } for(i = 0; i < items; i++) { int list_type_byte = buffer_save_space(buffer, 1); char name[16]; PyObject* item_value; if (list_type_byte == -1) { PyErr_NoMemory(); return 0; } INT2STRING(name, (int)i); if (!buffer_write_bytes(buffer, name, (int)strlen(name) + 1)) { return 0; } if (!(item_value = PySequence_GetItem(value, i))) return 0; if (!write_element_to_buffer(self, buffer, list_type_byte, item_value, check_keys, options)) { Py_DECREF(item_value); return 0; } Py_DECREF(item_value); } /* write null byte and fill in length */ if (!buffer_write_bytes(buffer, &zero, 1)) { return 0; } length = buffer_get_position(buffer) - start_position; buffer_write_int32_at_position( buffer, length_location, (int32_t)length); return 1; #if PY_MAJOR_VERSION >= 3 /* Python3 special case. Store bytes as BSON binary subtype 0. */ } else if (PyBytes_Check(value)) { char subtype = 0; int size; const char* data = PyBytes_AS_STRING(value); if (!data) return 0; if ((size = _downcast_and_check(PyBytes_GET_SIZE(value), 0)) == -1) return 0; *(buffer_get_buffer(buffer) + type_byte) = 0x05; if (!buffer_write_int32(buffer, (int32_t)size)) { return 0; } if (!buffer_write_bytes(buffer, &subtype, 1)) { return 0; } if (!buffer_write_bytes(buffer, data, size)) { return 0; } return 1; #else /* PyString_Check only works in Python 2.x. */ } else if (PyString_Check(value)) { result_t status; const char* data; int size; if (!(data = PyString_AS_STRING(value))) return 0; if ((size = _downcast_and_check(PyString_GET_SIZE(value), 1)) == -1) return 0; *(buffer_get_buffer(buffer) + type_byte) = 0x02; status = check_string((const unsigned char*)data, size - 1, 1, 0); if (status == NOT_UTF_8) { PyObject* InvalidStringData = _error("InvalidStringData"); if (InvalidStringData) { PyObject* repr = PyObject_Repr(value); char* repr_as_cstr = repr ? PyString_AsString(repr) : NULL; if (repr_as_cstr) { PyObject *message = PyString_FromFormat( "strings in documents must be valid UTF-8: %s", repr_as_cstr); if (message) { PyErr_SetObject(InvalidStringData, message); Py_DECREF(message); } } else { /* repr(value) failed, use a generic message. */ PyErr_SetString( InvalidStringData, "strings in documents must be valid UTF-8"); } Py_XDECREF(repr); Py_DECREF(InvalidStringData); } return 0; } if (!buffer_write_int32(buffer, (int32_t)size)) { return 0; } if (!buffer_write_bytes(buffer, data, size)) { return 0; } return 1; #endif } else if (PyUnicode_Check(value)) { *(buffer_get_buffer(buffer) + type_byte) = 0x02; return write_unicode(buffer, value); } else if (PyDateTime_Check(value)) { long long millis; PyObject* utcoffset = PyObject_CallMethod(value, "utcoffset", NULL); if (utcoffset == NULL) return 0; if (utcoffset != Py_None) { PyObject* result = PyNumber_Subtract(value, utcoffset); Py_DECREF(utcoffset); if (!result) { return 0; } millis = millis_from_datetime(result); Py_DECREF(result); } else { millis = millis_from_datetime(value); } *(buffer_get_buffer(buffer) + type_byte) = 0x09; return buffer_write_int64(buffer, (int64_t)millis); } else if (PyObject_TypeCheck(value, state->REType)) { return _write_regex_to_buffer(buffer, type_byte, value); } /* * Try Mapping and UUID last since we have to import * them if we're in a sub-interpreter. */ #if PY_MAJOR_VERSION >= 3 mapping_type = _get_object(state->Mapping, "collections.abc", "Mapping"); #else mapping_type = _get_object(state->Mapping, "collections", "Mapping"); #endif if (mapping_type && PyObject_IsInstance(value, mapping_type)) { Py_DECREF(mapping_type); /* PyObject_IsInstance returns -1 on error */ if (PyErr_Occurred()) { return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x03; return write_dict(self, buffer, value, check_keys, options, 0); } uuid_type = _get_object(state->UUID, "uuid", "UUID"); if (uuid_type && PyObject_IsInstance(value, uuid_type)) { /* Just a special case of Binary above, but * simpler to do as a separate case. */ PyObject* bytes; /* Could be bytes, bytearray, str... */ const char* data; /* UUID is always 16 bytes */ int size = 16; char subtype; Py_DECREF(uuid_type); /* PyObject_IsInstance returns -1 on error */ if (PyErr_Occurred()) { return 0; } if (options->uuid_rep == JAVA_LEGACY || options->uuid_rep == CSHARP_LEGACY) { subtype = 3; } else { subtype = options->uuid_rep; } *(buffer_get_buffer(buffer) + type_byte) = 0x05; if (!buffer_write_int32(buffer, (int32_t)size)) { return 0; } if (!buffer_write_bytes(buffer, &subtype, 1)) { return 0; } if (options->uuid_rep == CSHARP_LEGACY) { /* Legacy C# byte order */ bytes = PyObject_GetAttrString(value, "bytes_le"); } else { bytes = PyObject_GetAttrString(value, "bytes"); } if (!bytes) { return 0; } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AsString(bytes); #else data = PyString_AsString(bytes); #endif if (data == NULL) { Py_DECREF(bytes); return 0; } if (options->uuid_rep == JAVA_LEGACY) { /* Store in legacy java byte order. */ char as_legacy_java[16]; _fix_java(data, as_legacy_java); if (!buffer_write_bytes(buffer, as_legacy_java, size)) { Py_DECREF(bytes); return 0; } } else { if (!buffer_write_bytes(buffer, data, size)) { Py_DECREF(bytes); return 0; } } Py_DECREF(bytes); return 1; } Py_XDECREF(mapping_type); Py_XDECREF(uuid_type); /* We can't determine value's type. Fail. */ _set_cannot_encode(value); return 0; } static int check_key_name(const char* name, int name_length) { if (name_length > 0 && name[0] == '$') { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromFormat( "key '%s' must not start with '$'", name); #else PyObject* errmsg = PyString_FromFormat( "key '%s' must not start with '$'", name); #endif if (errmsg) { PyErr_SetObject(InvalidDocument, errmsg); Py_DECREF(errmsg); } Py_DECREF(InvalidDocument); } return 0; } if (strchr(name, '.')) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromFormat( "key '%s' must not contain '.'", name); #else PyObject* errmsg = PyString_FromFormat( "key '%s' must not contain '.'", name); #endif if (errmsg) { PyErr_SetObject(InvalidDocument, errmsg); Py_DECREF(errmsg); } Py_DECREF(InvalidDocument); } return 0; } return 1; } /* Write a (key, value) pair to the buffer. * * Returns 0 on failure */ int write_pair(PyObject* self, buffer_t buffer, const char* name, int name_length, PyObject* value, unsigned char check_keys, const codec_options_t* options, unsigned char allow_id) { int type_byte; /* Don't write any _id elements unless we're explicitly told to - * _id has to be written first so we do so, but don't bother * deleting it from the dictionary being written. */ if (!allow_id && strcmp(name, "_id") == 0) { return 1; } type_byte = buffer_save_space(buffer, 1); if (type_byte == -1) { PyErr_NoMemory(); return 0; } if (check_keys && !check_key_name(name, name_length)) { return 0; } if (!buffer_write_bytes(buffer, name, name_length + 1)) { return 0; } if (!write_element_to_buffer(self, buffer, type_byte, value, check_keys, options)) { return 0; } return 1; } int decode_and_write_pair(PyObject* self, buffer_t buffer, PyObject* key, PyObject* value, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) { PyObject* encoded; const char* data; int size; if (PyUnicode_Check(key)) { encoded = PyUnicode_AsUTF8String(key); if (!encoded) { return 0; } #if PY_MAJOR_VERSION >= 3 if (!(data = PyBytes_AS_STRING(encoded))) { Py_DECREF(encoded); return 0; } if ((size = _downcast_and_check(PyBytes_GET_SIZE(encoded), 1)) == -1) { Py_DECREF(encoded); return 0; } #else if (!(data = PyString_AS_STRING(encoded))) { Py_DECREF(encoded); return 0; } if ((size = _downcast_and_check(PyString_GET_SIZE(encoded), 1)) == -1) { Py_DECREF(encoded); return 0; } #endif if (strlen(data) != (size_t)(size - 1)) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyErr_SetString(InvalidDocument, "Key names must not contain the NULL byte"); Py_DECREF(InvalidDocument); } Py_DECREF(encoded); return 0; } #if PY_MAJOR_VERSION < 3 } else if (PyString_Check(key)) { result_t status; encoded = key; Py_INCREF(encoded); if (!(data = PyString_AS_STRING(encoded))) { Py_DECREF(encoded); return 0; } if ((size = _downcast_and_check(PyString_GET_SIZE(encoded), 1)) == -1) { Py_DECREF(encoded); return 0; } status = check_string((const unsigned char*)data, size - 1, 1, 1); if (status == NOT_UTF_8) { PyObject* InvalidStringData = _error("InvalidStringData"); if (InvalidStringData) { PyErr_SetString(InvalidStringData, "strings in documents must be valid UTF-8"); Py_DECREF(InvalidStringData); } Py_DECREF(encoded); return 0; } else if (status == HAS_NULL) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyErr_SetString(InvalidDocument, "Key names must not contain the NULL byte"); Py_DECREF(InvalidDocument); } Py_DECREF(encoded); return 0; } #endif } else { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyObject* repr = PyObject_Repr(key); if (repr) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromString( "documents must have only string keys, key was "); #else PyObject* errmsg = PyString_FromString( "documents must have only string keys, key was "); #endif if (errmsg) { #if PY_MAJOR_VERSION >= 3 PyObject* error = PyUnicode_Concat(errmsg, repr); if (error) { PyErr_SetObject(InvalidDocument, error); Py_DECREF(error); } Py_DECREF(errmsg); Py_DECREF(repr); #else PyString_ConcatAndDel(&errmsg, repr); if (errmsg) { PyErr_SetObject(InvalidDocument, errmsg); Py_DECREF(errmsg); } #endif } else { Py_DECREF(repr); } } Py_DECREF(InvalidDocument); } return 0; } /* If top_level is True, don't allow writing _id here - it was already written. */ if (!write_pair(self, buffer, data, size - 1, value, check_keys, options, !top_level)) { Py_DECREF(encoded); return 0; } Py_DECREF(encoded); return 1; } /* returns 0 on failure */ int write_dict(PyObject* self, buffer_t buffer, PyObject* dict, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) { PyObject* key; PyObject* iter; char zero = 0; int length; int length_location; struct module_state *state = GETSTATE(self); #if PY_MAJOR_VERSION >= 3 PyObject* mapping_type = _get_object(state->Mapping, "collections.abc", "Mapping"); #else PyObject* mapping_type = _get_object(state->Mapping, "collections", "Mapping"); #endif if (mapping_type) { if (!PyObject_IsInstance(dict, mapping_type)) { PyObject* repr; Py_DECREF(mapping_type); if ((repr = PyObject_Repr(dict))) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromString( "encoder expected a mapping type but got: "); if (errmsg) { PyObject* error = PyUnicode_Concat(errmsg, repr); if (error) { PyErr_SetObject(PyExc_TypeError, error); Py_DECREF(error); } Py_DECREF(errmsg); Py_DECREF(repr); } #else PyObject* errmsg = PyString_FromString( "encoder expected a mapping type but got: "); if (errmsg) { PyString_ConcatAndDel(&errmsg, repr); if (errmsg) { PyErr_SetObject(PyExc_TypeError, errmsg); Py_DECREF(errmsg); } } #endif else { Py_DECREF(repr); } } else { PyErr_SetString(PyExc_TypeError, "encoder expected a mapping type"); } return 0; } Py_DECREF(mapping_type); /* PyObject_IsInstance returns -1 on error */ if (PyErr_Occurred()) { return 0; } } length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyErr_NoMemory(); return 0; } /* Write _id first if this is a top level doc. */ if (top_level) { /* * If "dict" is a defaultdict we don't want to call * PyMapping_GetItemString on it. That would **create** * an _id where one didn't previously exist (PYTHON-871). */ if (PyDict_Check(dict)) { /* PyDict_GetItemString returns a borrowed reference. */ PyObject* _id = PyDict_GetItemString(dict, "_id"); if (_id) { if (!write_pair(self, buffer, "_id", 3, _id, check_keys, options, 1)) { return 0; } } } else if (PyMapping_HasKeyString(dict, "_id")) { PyObject* _id = PyMapping_GetItemString(dict, "_id"); if (!_id) { return 0; } if (!write_pair(self, buffer, "_id", 3, _id, check_keys, options, 1)) { Py_DECREF(_id); return 0; } /* PyMapping_GetItemString returns a new reference. */ Py_DECREF(_id); } } iter = PyObject_GetIter(dict); if (iter == NULL) { return 0; } while ((key = PyIter_Next(iter)) != NULL) { PyObject* value = PyObject_GetItem(dict, key); if (!value) { PyErr_SetObject(PyExc_KeyError, key); Py_DECREF(key); Py_DECREF(iter); return 0; } if (!decode_and_write_pair(self, buffer, key, value, check_keys, options, top_level)) { Py_DECREF(key); Py_DECREF(value); Py_DECREF(iter); return 0; } Py_DECREF(key); Py_DECREF(value); } Py_DECREF(iter); if (PyErr_Occurred()) { return 0; } /* write null byte and fill in length */ if (!buffer_write_bytes(buffer, &zero, 1)) { return 0; } length = buffer_get_position(buffer) - length_location; buffer_write_int32_at_position( buffer, length_location, (int32_t)length); return 1; } static PyObject* _cbson_dict_to_bson(PyObject* self, PyObject* args) { PyObject* dict; PyObject* result; unsigned char check_keys; unsigned char top_level = 1; codec_options_t options; buffer_t buffer; PyObject* raw_bson_document_bytes_obj; long type_marker; if (!PyArg_ParseTuple(args, "ObO&|b", &dict, &check_keys, convert_codec_options, &options, &top_level)) { return NULL; } /* check for RawBSONDocument */ type_marker = _type_marker(dict); if (type_marker < 0) { destroy_codec_options(&options); return NULL; } else if (101 == type_marker) { destroy_codec_options(&options); raw_bson_document_bytes_obj = PyObject_GetAttrString(dict, "raw"); if (NULL == raw_bson_document_bytes_obj) { return NULL; } return raw_bson_document_bytes_obj; } buffer = buffer_new(); if (!buffer) { destroy_codec_options(&options); PyErr_NoMemory(); return NULL; } if (!write_dict(self, buffer, dict, check_keys, &options, top_level)) { destroy_codec_options(&options); buffer_free(buffer); return NULL; } /* objectify buffer */ #if PY_MAJOR_VERSION >= 3 result = Py_BuildValue("y#", buffer_get_buffer(buffer), buffer_get_position(buffer)); #else result = Py_BuildValue("s#", buffer_get_buffer(buffer), buffer_get_position(buffer)); #endif destroy_codec_options(&options); buffer_free(buffer); return result; } static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, unsigned* position, unsigned char type, unsigned max, const codec_options_t* options) { struct module_state *state = GETSTATE(self); PyObject* value = NULL; switch (type) { case 1: { double d; if (max < 8) { goto invalid; } memcpy(&d, buffer + *position, 8); value = PyFloat_FromDouble(BSON_DOUBLE_FROM_LE(d)); *position += 8; break; } case 2: case 14: { uint32_t value_length; if (max < 4) { goto invalid; } memcpy(&value_length, buffer + *position, 4); value_length = BSON_UINT32_FROM_LE(value_length); /* Encoded string length + string */ if (!value_length || max < value_length || max < 4 + value_length) { goto invalid; } *position += 4; /* Strings must end in \0 */ if (buffer[*position + value_length - 1]) { goto invalid; } value = PyUnicode_DecodeUTF8( buffer + *position, value_length - 1, options->unicode_decode_error_handler); if (!value) { goto invalid; } *position += value_length; break; } case 3: { PyObject* collection; uint32_t size; if (max < 4) { goto invalid; } memcpy(&size, buffer + *position, 4); size = BSON_UINT32_FROM_LE(size); if (size < BSON_MIN_SIZE || max < size) { goto invalid; } /* Check for bad eoo */ if (buffer[*position + size - 1]) { goto invalid; } if (options->is_raw_bson) { value = PyObject_CallFunction( options->document_class, BYTES_FORMAT_STRING "O", buffer + *position, size, options->options_obj); if (!value) { goto invalid; } *position += size; break; } value = elements_to_dict(self, buffer + *position + 4, size - 5, options); if (!value) { goto invalid; } /* Decoding for DBRefs */ if (PyMapping_HasKeyString(value, "$ref")) { /* DBRef */ PyObject* dbref = NULL; PyObject* dbref_type; PyObject* id; PyObject* database; collection = PyMapping_GetItemString(value, "$ref"); /* PyMapping_GetItemString returns NULL to indicate error. */ if (!collection) { goto invalid; } PyMapping_DelItemString(value, "$ref"); if (PyMapping_HasKeyString(value, "$id")) { id = PyMapping_GetItemString(value, "$id"); if (!id) { Py_DECREF(collection); goto invalid; } PyMapping_DelItemString(value, "$id"); } else { id = Py_None; Py_INCREF(id); } if (PyMapping_HasKeyString(value, "$db")) { database = PyMapping_GetItemString(value, "$db"); if (!database) { Py_DECREF(collection); Py_DECREF(id); goto invalid; } PyMapping_DelItemString(value, "$db"); } else { database = Py_None; Py_INCREF(database); } if ((dbref_type = _get_object(state->DBRef, "bson.dbref", "DBRef"))) { dbref = PyObject_CallFunctionObjArgs(dbref_type, collection, id, database, value, NULL); Py_DECREF(dbref_type); } Py_DECREF(value); value = dbref; Py_DECREF(id); Py_DECREF(collection); Py_DECREF(database); } *position += size; break; } case 4: { uint32_t size, end; if (max < 4) { goto invalid; } memcpy(&size, buffer + *position, 4); size = BSON_UINT32_FROM_LE(size); if (size < BSON_MIN_SIZE || max < size) { goto invalid; } end = *position + size - 1; /* Check for bad eoo */ if (buffer[end]) { goto invalid; } *position += 4; value = PyList_New(0); if (!value) { goto invalid; } while (*position < end) { PyObject* to_append; unsigned char bson_type = (unsigned char)buffer[(*position)++]; size_t key_size = strlen(buffer + *position); if (max < key_size) { Py_DECREF(value); goto invalid; } /* just skip the key, they're in order. */ *position += (unsigned)key_size + 1; if (Py_EnterRecursiveCall(" while decoding a list value")) { Py_DECREF(value); goto invalid; } to_append = get_value(self, name, buffer, position, bson_type, max - (unsigned)key_size, options); Py_LeaveRecursiveCall(); if (!to_append) { Py_DECREF(value); goto invalid; } if (PyList_Append(value, to_append) < 0) { Py_DECREF(value); Py_DECREF(to_append); goto invalid; } Py_DECREF(to_append); } if (*position != end) { goto invalid; } (*position)++; break; } case 5: { PyObject* data; PyObject* st; PyObject* type_to_create; uint32_t length, length2; unsigned char subtype; if (max < 5) { goto invalid; } memcpy(&length, buffer + *position, 4); length = BSON_UINT32_FROM_LE(length); if (max < length) { goto invalid; } subtype = (unsigned char)buffer[*position + 4]; *position += 5; if (subtype == 2) { if (length < 4) { goto invalid; } memcpy(&length2, buffer + *position, 4); length2 = BSON_UINT32_FROM_LE(length2); if (length2 != length - 4) { goto invalid; } } #if PY_MAJOR_VERSION >= 3 /* Python3 special case. Decode BSON binary subtype 0 to bytes. */ if (subtype == 0) { value = PyBytes_FromStringAndSize(buffer + *position, length); *position += length; break; } if (subtype == 2) { data = PyBytes_FromStringAndSize(buffer + *position + 4, length - 4); } else { data = PyBytes_FromStringAndSize(buffer + *position, length); } #else if (subtype == 2) { data = PyString_FromStringAndSize(buffer + *position + 4, length - 4); } else { data = PyString_FromStringAndSize(buffer + *position, length); } #endif if (!data) { goto invalid; } /* Encode as UUID, not Binary */ if (subtype == 3 || subtype == 4) { PyObject* kwargs; PyObject* args = PyTuple_New(0); /* UUID should always be 16 bytes */ if (!args || length != 16) { Py_DECREF(data); goto invalid; } kwargs = PyDict_New(); if (!kwargs) { Py_DECREF(data); Py_DECREF(args); goto invalid; } /* * From this point, we hold refs to args, kwargs, and data. * If anything fails, goto uuiderror to clean them up. */ if (subtype == 3 && options->uuid_rep == CSHARP_LEGACY) { /* Legacy C# byte order */ if ((PyDict_SetItemString(kwargs, "bytes_le", data)) == -1) goto uuiderror; } else { if (subtype == 3 && options->uuid_rep == JAVA_LEGACY) { /* Convert from legacy java byte order */ char big_endian[16]; _fix_java(buffer + *position, big_endian); /* Free the previously created PyString object */ Py_DECREF(data); #if PY_MAJOR_VERSION >= 3 data = PyBytes_FromStringAndSize(big_endian, length); #else data = PyString_FromStringAndSize(big_endian, length); #endif if (data == NULL) goto uuiderror; } if ((PyDict_SetItemString(kwargs, "bytes", data)) == -1) goto uuiderror; } if ((type_to_create = _get_object(state->UUID, "uuid", "UUID"))) { value = PyObject_Call(type_to_create, args, kwargs); Py_DECREF(type_to_create); } Py_DECREF(args); Py_DECREF(kwargs); Py_DECREF(data); if (!value) { goto invalid; } *position += length; break; uuiderror: Py_DECREF(args); Py_DECREF(kwargs); Py_XDECREF(data); goto invalid; } #if PY_MAJOR_VERSION >= 3 st = PyLong_FromLong(subtype); #else st = PyInt_FromLong(subtype); #endif if (!st) { Py_DECREF(data); goto invalid; } if ((type_to_create = _get_object(state->Binary, "bson.binary", "Binary"))) { value = PyObject_CallFunctionObjArgs(type_to_create, data, st, NULL); Py_DECREF(type_to_create); } Py_DECREF(st); Py_DECREF(data); if (!value) { goto invalid; } *position += length; break; } case 6: case 10: { value = Py_None; Py_INCREF(value); break; } case 7: { PyObject* objectid_type; if (max < 12) { goto invalid; } if ((objectid_type = _get_object(state->ObjectId, "bson.objectid", "ObjectId"))) { #if PY_MAJOR_VERSION >= 3 value = PyObject_CallFunction(objectid_type, "y#", buffer + *position, 12); #else value = PyObject_CallFunction(objectid_type, "s#", buffer + *position, 12); #endif Py_DECREF(objectid_type); } *position += 12; break; } case 8: { char boolean_raw = buffer[(*position)++]; if (0 == boolean_raw) { value = Py_False; } else if (1 == boolean_raw) { value = Py_True; } else { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_Format(InvalidBSON, "invalid boolean value: %x", boolean_raw); Py_DECREF(InvalidBSON); } return NULL; } Py_INCREF(value); break; } case 9: { PyObject* utc_type; PyObject* naive; PyObject* replace; PyObject* args; PyObject* kwargs; PyObject* astimezone; int64_t millis; if (max < 8) { goto invalid; } memcpy(&millis, buffer + *position, 8); millis = (int64_t)BSON_UINT64_FROM_LE(millis); naive = datetime_from_millis(millis); *position += 8; if (!options->tz_aware) { /* In the naive case, we're done here. */ value = naive; break; } if (!naive) { goto invalid; } replace = PyObject_GetAttrString(naive, "replace"); Py_DECREF(naive); if (!replace) { goto invalid; } args = PyTuple_New(0); if (!args) { Py_DECREF(replace); goto invalid; } kwargs = PyDict_New(); if (!kwargs) { Py_DECREF(replace); Py_DECREF(args); goto invalid; } utc_type = _get_object(state->UTC, "bson.tz_util", "utc"); if (!utc_type || PyDict_SetItemString(kwargs, "tzinfo", utc_type) == -1) { Py_DECREF(replace); Py_DECREF(args); Py_DECREF(kwargs); Py_XDECREF(utc_type); goto invalid; } Py_XDECREF(utc_type); value = PyObject_Call(replace, args, kwargs); if (!value) { Py_DECREF(replace); Py_DECREF(args); Py_DECREF(kwargs); goto invalid; } /* convert to local time */ if (options->tzinfo != Py_None) { astimezone = PyObject_GetAttrString(value, "astimezone"); if (!astimezone) { Py_DECREF(replace); Py_DECREF(args); Py_DECREF(kwargs); goto invalid; } value = PyObject_CallFunctionObjArgs(astimezone, options->tzinfo, NULL); Py_DECREF(astimezone); } Py_DECREF(replace); Py_DECREF(args); Py_DECREF(kwargs); break; } case 11: { PyObject* regex_class; PyObject* pattern; int flags; size_t flags_length, i; size_t pattern_length = strlen(buffer + *position); if (pattern_length > BSON_MAX_SIZE || max < pattern_length) { goto invalid; } pattern = PyUnicode_DecodeUTF8( buffer + *position, pattern_length, options->unicode_decode_error_handler); if (!pattern) { goto invalid; } *position += (unsigned)pattern_length + 1; flags_length = strlen(buffer + *position); if (flags_length > BSON_MAX_SIZE || (BSON_MAX_SIZE - pattern_length) < flags_length) { Py_DECREF(pattern); goto invalid; } if (max < pattern_length + flags_length) { Py_DECREF(pattern); goto invalid; } flags = 0; for (i = 0; i < flags_length; i++) { if (buffer[*position + i] == 'i') { flags |= 2; } else if (buffer[*position + i] == 'l') { flags |= 4; } else if (buffer[*position + i] == 'm') { flags |= 8; } else if (buffer[*position + i] == 's') { flags |= 16; } else if (buffer[*position + i] == 'u') { flags |= 32; } else if (buffer[*position + i] == 'x') { flags |= 64; } } *position += (unsigned)flags_length + 1; regex_class = _get_object(state->Regex, "bson.regex", "Regex"); if (regex_class) { value = PyObject_CallFunction(regex_class, "Oi", pattern, flags); Py_DECREF(regex_class); } Py_DECREF(pattern); break; } case 12: { uint32_t coll_length; PyObject* collection; PyObject* id = NULL; PyObject* objectid_type; PyObject* dbref_type; if (max < 4) { goto invalid; } memcpy(&coll_length, buffer + *position, 4); coll_length = BSON_UINT32_FROM_LE(coll_length); /* Encoded string length + string + 12 byte ObjectId */ if (!coll_length || max < coll_length || max < 4 + coll_length + 12) { goto invalid; } *position += 4; /* Strings must end in \0 */ if (buffer[*position + coll_length - 1]) { goto invalid; } collection = PyUnicode_DecodeUTF8( buffer + *position, coll_length - 1, options->unicode_decode_error_handler); if (!collection) { goto invalid; } *position += coll_length; if ((objectid_type = _get_object(state->ObjectId, "bson.objectid", "ObjectId"))) { #if PY_MAJOR_VERSION >= 3 id = PyObject_CallFunction(objectid_type, "y#", buffer + *position, 12); #else id = PyObject_CallFunction(objectid_type, "s#", buffer + *position, 12); #endif Py_DECREF(objectid_type); } if (!id) { Py_DECREF(collection); goto invalid; } *position += 12; if ((dbref_type = _get_object(state->DBRef, "bson.dbref", "DBRef"))) { value = PyObject_CallFunctionObjArgs(dbref_type, collection, id, NULL); Py_DECREF(dbref_type); } Py_DECREF(collection); Py_DECREF(id); break; } case 13: { PyObject* code; PyObject* code_type; uint32_t value_length; if (max < 4) { goto invalid; } memcpy(&value_length, buffer + *position, 4); value_length = BSON_UINT32_FROM_LE(value_length); /* Encoded string length + string */ if (!value_length || max < value_length || max < 4 + value_length) { goto invalid; } *position += 4; /* Strings must end in \0 */ if (buffer[*position + value_length - 1]) { goto invalid; } code = PyUnicode_DecodeUTF8( buffer + *position, value_length - 1, options->unicode_decode_error_handler); if (!code) { goto invalid; } *position += value_length; if ((code_type = _get_object(state->Code, "bson.code", "Code"))) { value = PyObject_CallFunctionObjArgs(code_type, code, NULL, NULL); Py_DECREF(code_type); } Py_DECREF(code); break; } case 15: { uint32_t c_w_s_size; uint32_t code_size; uint32_t scope_size; PyObject* code; PyObject* scope; PyObject* code_type; if (max < 8) { goto invalid; } memcpy(&c_w_s_size, buffer + *position, 4); c_w_s_size = BSON_UINT32_FROM_LE(c_w_s_size); *position += 4; if (max < c_w_s_size) { goto invalid; } memcpy(&code_size, buffer + *position, 4); code_size = BSON_UINT32_FROM_LE(code_size); /* code_w_scope length + code length + code + scope length */ if (!code_size || max < code_size || max < 4 + 4 + code_size + 4) { goto invalid; } *position += 4; /* Strings must end in \0 */ if (buffer[*position + code_size - 1]) { goto invalid; } code = PyUnicode_DecodeUTF8( buffer + *position, code_size - 1, options->unicode_decode_error_handler); if (!code) { goto invalid; } *position += code_size; memcpy(&scope_size, buffer + *position, 4); scope_size = BSON_UINT32_FROM_LE(scope_size); if (scope_size < BSON_MIN_SIZE) { Py_DECREF(code); goto invalid; } /* code length + code + scope length + scope */ if ((4 + code_size + 4 + scope_size) != c_w_s_size) { Py_DECREF(code); goto invalid; } /* Check for bad eoo */ if (buffer[*position + scope_size - 1]) { goto invalid; } scope = elements_to_dict(self, buffer + *position + 4, scope_size - 5, options); if (!scope) { Py_DECREF(code); goto invalid; } *position += scope_size; if ((code_type = _get_object(state->Code, "bson.code", "Code"))) { value = PyObject_CallFunctionObjArgs(code_type, code, scope, NULL); Py_DECREF(code_type); } Py_DECREF(code); Py_DECREF(scope); break; } case 16: { int32_t i; if (max < 4) { goto invalid; } memcpy(&i, buffer + *position, 4); i = (int32_t)BSON_UINT32_FROM_LE(i); #if PY_MAJOR_VERSION >= 3 value = PyLong_FromLong(i); #else value = PyInt_FromLong(i); #endif if (!value) { goto invalid; } *position += 4; break; } case 17: { uint32_t time, inc; PyObject* timestamp_type; if (max < 8) { goto invalid; } memcpy(&inc, buffer + *position, 4); memcpy(&time, buffer + *position + 4, 4); inc = BSON_UINT32_FROM_LE(inc); time = BSON_UINT32_FROM_LE(time); if ((timestamp_type = _get_object(state->Timestamp, "bson.timestamp", "Timestamp"))) { value = PyObject_CallFunction(timestamp_type, "II", time, inc); Py_DECREF(timestamp_type); } *position += 8; break; } case 18: { int64_t ll; PyObject* bson_int64_type = _get_object(state->BSONInt64, "bson.int64", "Int64"); if (!bson_int64_type) goto invalid; if (max < 8) { Py_DECREF(bson_int64_type); goto invalid; } memcpy(&ll, buffer + *position, 8); ll = (int64_t)BSON_UINT64_FROM_LE(ll); value = PyObject_CallFunction(bson_int64_type, "L", ll); *position += 8; Py_DECREF(bson_int64_type); break; } case 19: { PyObject* dec128; if (max < 16) { goto invalid; } if ((dec128 = _get_object(state->Decimal128, "bson.decimal128", "Decimal128"))) { value = PyObject_CallMethod(dec128, "from_bid", #if PY_MAJOR_VERSION >= 3 "y#", #else "s#", #endif buffer + *position, 16); Py_DECREF(dec128); } *position += 16; break; } case 255: { PyObject* minkey_type = _get_object(state->MinKey, "bson.min_key", "MinKey"); if (!minkey_type) goto invalid; value = PyObject_CallFunctionObjArgs(minkey_type, NULL); Py_DECREF(minkey_type); break; } case 127: { PyObject* maxkey_type = _get_object(state->MaxKey, "bson.max_key", "MaxKey"); if (!maxkey_type) goto invalid; value = PyObject_CallFunctionObjArgs(maxkey_type, NULL); Py_DECREF(maxkey_type); break; } default: { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyObject* bobj = PyBytes_FromFormat("%c", type); if (bobj) { PyObject* repr = PyObject_Repr(bobj); Py_DECREF(bobj); /* * See http://bugs.python.org/issue22023 for why we can't * just use PyUnicode_FromFormat with %S or %R to do this * work. */ if (repr) { PyObject* left = PyUnicode_FromString( "Detected unknown BSON type "); if (left) { PyObject* lmsg = PyUnicode_Concat(left, repr); Py_DECREF(left); if (lmsg) { PyObject* errmsg = PyUnicode_FromFormat( "%U for fieldname '%U'. Are you using the " "latest driver version?", lmsg, name); if (errmsg) { PyErr_SetObject(InvalidBSON, errmsg); Py_DECREF(errmsg); } Py_DECREF(lmsg); } } Py_DECREF(repr); } } Py_DECREF(InvalidBSON); } goto invalid; } } if (value) { return value; } invalid: /* * Wrap any non-InvalidBSON errors in InvalidBSON. */ if (PyErr_Occurred()) { PyObject *etype, *evalue, *etrace; PyObject *InvalidBSON; /* * Calling _error clears the error state, so fetch it first. */ PyErr_Fetch(&etype, &evalue, &etrace); /* Dont reraise anything but PyExc_Exceptions as InvalidBSON. */ if (PyErr_GivenExceptionMatches(etype, PyExc_Exception)) { InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { if (!PyErr_GivenExceptionMatches(etype, InvalidBSON)) { /* * Raise InvalidBSON(str(e)). */ Py_DECREF(etype); etype = InvalidBSON; if (evalue) { PyObject *msg = PyObject_Str(evalue); Py_DECREF(evalue); evalue = msg; } PyErr_NormalizeException(&etype, &evalue, &etrace); } else { /* * The current exception matches InvalidBSON, so we don't * need this reference after all. */ Py_DECREF(InvalidBSON); } } } /* Steals references to args. */ PyErr_Restore(etype, evalue, etrace); } else { PyObject *InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "invalid length or type code"); Py_DECREF(InvalidBSON); } } return NULL; } /* * Get the next 'name' and 'value' from a document in a string, whose position * is provided. * * Returns the position of the next element in the document, or -1 on error. */ static int _element_to_dict(PyObject* self, const char* string, unsigned position, unsigned max, const codec_options_t* options, PyObject** name, PyObject** value) { unsigned char type = (unsigned char)string[position++]; size_t name_length = strlen(string + position); if (name_length > BSON_MAX_SIZE || position + name_length >= max) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetNone(InvalidBSON); Py_DECREF(InvalidBSON); } return -1; } *name = PyUnicode_DecodeUTF8( string + position, name_length, options->unicode_decode_error_handler); if (!*name) { /* If NULL is returned then wrap the UnicodeDecodeError in an InvalidBSON error */ PyObject *etype, *evalue, *etrace; PyObject *InvalidBSON; PyErr_Fetch(&etype, &evalue, &etrace); if (PyErr_GivenExceptionMatches(etype, PyExc_Exception)) { InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { Py_DECREF(etype); etype = InvalidBSON; if (evalue) { PyObject *msg = PyObject_Str(evalue); Py_DECREF(evalue); evalue = msg; } PyErr_NormalizeException(&etype, &evalue, &etrace); } } PyErr_Restore(etype, evalue, etrace); return -1; } position += (unsigned)name_length + 1; *value = get_value(self, *name, string, &position, type, max - position, options); if (!*value) { Py_DECREF(*name); return -1; } return position; } static PyObject* _cbson_element_to_dict(PyObject* self, PyObject* args) { char* string; PyObject* bson; codec_options_t options; unsigned position; unsigned max; int new_position; PyObject* name; PyObject* value; PyObject* result_tuple; if (!PyArg_ParseTuple(args, "OII|O&", &bson, &position, &max, convert_codec_options, &options)) { return NULL; } if (PyTuple_GET_SIZE(args) < 4) { if (!default_codec_options(GETSTATE(self), &options)) { return NULL; } } #if PY_MAJOR_VERSION >= 3 if (!PyBytes_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _element_to_dict must be a bytes object"); #else if (!PyString_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _element_to_dict must be a string"); #endif return NULL; } #if PY_MAJOR_VERSION >= 3 string = PyBytes_AS_STRING(bson); #else string = PyString_AS_STRING(bson); #endif new_position = _element_to_dict(self, string, position, max, &options, &name, &value); if (new_position < 0) { return NULL; } result_tuple = Py_BuildValue("NNi", name, value, new_position); if (!result_tuple) { Py_DECREF(name); Py_DECREF(value); return NULL; } return result_tuple; } static PyObject* _elements_to_dict(PyObject* self, const char* string, unsigned max, const codec_options_t* options) { unsigned position = 0; PyObject* dict = PyObject_CallObject(options->document_class, NULL); if (!dict) { return NULL; } while (position < max) { PyObject* name = NULL; PyObject* value = NULL; int new_position; new_position = _element_to_dict( self, string, position, max, options, &name, &value); if (new_position < 0) { Py_DECREF(dict); return NULL; } else { position = (unsigned)new_position; } PyObject_SetItem(dict, name, value); Py_DECREF(name); Py_DECREF(value); } return dict; } static PyObject* elements_to_dict(PyObject* self, const char* string, unsigned max, const codec_options_t* options) { PyObject* result; if (Py_EnterRecursiveCall(" while decoding a BSON document")) return NULL; result = _elements_to_dict(self, string, max, options); Py_LeaveRecursiveCall(); return result; } static PyObject* _cbson_bson_to_dict(PyObject* self, PyObject* args) { int32_t size; Py_ssize_t total_size; const char* string; PyObject* bson; codec_options_t options; PyObject* result; PyObject* options_obj; if (! (PyArg_ParseTuple(args, "OO", &bson, &options_obj) && convert_codec_options(options_obj, &options))) { return NULL; } #if PY_MAJOR_VERSION >= 3 if (!PyBytes_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _bson_to_dict must be a bytes object"); #else if (!PyString_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _bson_to_dict must be a string"); #endif destroy_codec_options(&options); return NULL; } #if PY_MAJOR_VERSION >= 3 total_size = PyBytes_Size(bson); #else total_size = PyString_Size(bson); #endif if (total_size < BSON_MIN_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "not enough data for a BSON document"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); return NULL; } #if PY_MAJOR_VERSION >= 3 string = PyBytes_AsString(bson); #else string = PyString_AsString(bson); #endif if (!string) { destroy_codec_options(&options); return NULL; } memcpy(&size, string, 4); size = (int32_t)BSON_UINT32_FROM_LE(size); if (size < BSON_MIN_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "invalid message size"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); return NULL; } if (total_size < size || total_size > BSON_MAX_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "objsize too large"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); return NULL; } if (size != total_size || string[size - 1]) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "bad eoo"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); return NULL; } /* No need to decode fields if using RawBSONDocument */ if (options.is_raw_bson) { return PyObject_CallFunction( options.document_class, BYTES_FORMAT_STRING "O", string, size, options_obj); } result = elements_to_dict(self, string + 4, (unsigned)size - 5, &options); destroy_codec_options(&options); return result; } static PyObject* _cbson_decode_all(PyObject* self, PyObject* args) { int32_t size; Py_ssize_t total_size; const char* string; PyObject* bson; PyObject* dict; PyObject* result; codec_options_t options; PyObject* options_obj; if (!PyArg_ParseTuple(args, "O|O", &bson, &options_obj)) { return NULL; } if (PyTuple_GET_SIZE(args) < 2) { if (!default_codec_options(GETSTATE(self), &options)) { return NULL; } } else if (!convert_codec_options(options_obj, &options)) { return NULL; } #if PY_MAJOR_VERSION >= 3 if (!PyBytes_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to decode_all must be a bytes object"); #else if (!PyString_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to decode_all must be a string"); #endif destroy_codec_options(&options); return NULL; } #if PY_MAJOR_VERSION >= 3 total_size = PyBytes_Size(bson); string = PyBytes_AsString(bson); #else total_size = PyString_Size(bson); string = PyString_AsString(bson); #endif if (!string) { destroy_codec_options(&options); return NULL; } if (!(result = PyList_New(0))) { destroy_codec_options(&options); return NULL; } while (total_size > 0) { if (total_size < BSON_MIN_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "not enough data for a BSON document"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); Py_DECREF(result); return NULL; } memcpy(&size, string, 4); size = (int32_t)BSON_UINT32_FROM_LE(size); if (size < BSON_MIN_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "invalid message size"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); Py_DECREF(result); return NULL; } if (total_size < size) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "objsize too large"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); Py_DECREF(result); return NULL; } if (string[size - 1]) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "bad eoo"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); Py_DECREF(result); return NULL; } /* No need to decode fields if using RawBSONDocument. */ if (options.is_raw_bson) { dict = PyObject_CallFunction( options.document_class, BYTES_FORMAT_STRING "O", string, size, options_obj); } else { dict = elements_to_dict(self, string + 4, (unsigned)size - 5, &options); } if (!dict) { Py_DECREF(result); destroy_codec_options(&options); return NULL; } if (PyList_Append(result, dict) < 0) { Py_DECREF(dict); Py_DECREF(result); destroy_codec_options(&options); return NULL; } Py_DECREF(dict); string += size; total_size -= size; } destroy_codec_options(&options); return result; } static PyMethodDef _CBSONMethods[] = { {"_dict_to_bson", _cbson_dict_to_bson, METH_VARARGS, "convert a dictionary to a string containing its BSON representation."}, {"_bson_to_dict", _cbson_bson_to_dict, METH_VARARGS, "convert a BSON string to a SON object."}, {"decode_all", _cbson_decode_all, METH_VARARGS, "convert binary data to a sequence of documents."}, {"_element_to_dict", _cbson_element_to_dict, METH_VARARGS, "Decode a single key, value pair."}, {NULL, NULL, 0, NULL} }; #if PY_MAJOR_VERSION >= 3 #define INITERROR return NULL static int _cbson_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->Binary); Py_VISIT(GETSTATE(m)->Code); Py_VISIT(GETSTATE(m)->ObjectId); Py_VISIT(GETSTATE(m)->DBRef); Py_VISIT(GETSTATE(m)->Regex); Py_VISIT(GETSTATE(m)->UUID); Py_VISIT(GETSTATE(m)->Timestamp); Py_VISIT(GETSTATE(m)->MinKey); Py_VISIT(GETSTATE(m)->MaxKey); Py_VISIT(GETSTATE(m)->UTC); Py_VISIT(GETSTATE(m)->REType); return 0; } static int _cbson_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->Binary); Py_CLEAR(GETSTATE(m)->Code); Py_CLEAR(GETSTATE(m)->ObjectId); Py_CLEAR(GETSTATE(m)->DBRef); Py_CLEAR(GETSTATE(m)->Regex); Py_CLEAR(GETSTATE(m)->UUID); Py_CLEAR(GETSTATE(m)->Timestamp); Py_CLEAR(GETSTATE(m)->MinKey); Py_CLEAR(GETSTATE(m)->MaxKey); Py_CLEAR(GETSTATE(m)->UTC); Py_CLEAR(GETSTATE(m)->REType); return 0; } static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_cbson", NULL, sizeof(struct module_state), _CBSONMethods, NULL, _cbson_traverse, _cbson_clear, NULL }; PyMODINIT_FUNC PyInit__cbson(void) #else #define INITERROR return PyMODINIT_FUNC init_cbson(void) #endif { PyObject *m; PyObject *c_api_object; static void *_cbson_API[_cbson_API_POINTER_COUNT]; PyDateTime_IMPORT; if (PyDateTimeAPI == NULL) { INITERROR; } /* Export C API */ _cbson_API[_cbson_buffer_write_bytes_INDEX] = (void *) buffer_write_bytes; _cbson_API[_cbson_write_dict_INDEX] = (void *) write_dict; _cbson_API[_cbson_write_pair_INDEX] = (void *) write_pair; _cbson_API[_cbson_decode_and_write_pair_INDEX] = (void *) decode_and_write_pair; _cbson_API[_cbson_convert_codec_options_INDEX] = (void *) convert_codec_options; _cbson_API[_cbson_destroy_codec_options_INDEX] = (void *) destroy_codec_options; _cbson_API[_cbson_buffer_write_double_INDEX] = (void *) buffer_write_double; _cbson_API[_cbson_buffer_write_int32_INDEX] = (void *) buffer_write_int32; _cbson_API[_cbson_buffer_write_int64_INDEX] = (void *) buffer_write_int64; _cbson_API[_cbson_buffer_write_int32_at_position_INDEX] = (void *) buffer_write_int32_at_position; #if PY_VERSION_HEX >= 0x03010000 /* PyCapsule is new in python 3.1 */ c_api_object = PyCapsule_New((void *) _cbson_API, "_cbson._C_API", NULL); #else c_api_object = PyCObject_FromVoidPtr((void *) _cbson_API, NULL); #endif if (c_api_object == NULL) INITERROR; #if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&moduledef); #else m = Py_InitModule("_cbson", _CBSONMethods); #endif if (m == NULL) { Py_DECREF(c_api_object); INITERROR; } /* Import several python objects */ if (_load_python_objects(m)) { Py_DECREF(c_api_object); #if PY_MAJOR_VERSION >= 3 Py_DECREF(m); #endif INITERROR; } if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) { Py_DECREF(c_api_object); #if PY_MAJOR_VERSION >= 3 Py_DECREF(m); #endif INITERROR; } #if PY_MAJOR_VERSION >= 3 return m; #endif } pymongo-3.6.1/bson/bson-stdint-win32.h0000644000076600000240000001763413245617773020003 0ustar shanestaff00000000000000// ISO C9x compliant stdint.h for Microsoft Visual Studio // Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 // // Copyright (c) 2006-2013 Alexander Chemeris // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // 3. Neither the name of the product nor the names of its contributors may // be used to endorse or promote products derived from this software // without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // /////////////////////////////////////////////////////////////////////////////// #ifndef _MSC_VER // [ #error "Use this header only with Microsoft Visual C++ compilers!" #endif // _MSC_VER ] #ifndef _MSC_STDINT_H_ // [ #define _MSC_STDINT_H_ #if _MSC_VER > 1000 #pragma once #endif #if _MSC_VER >= 1600 // [ #include #else // ] _MSC_VER >= 1600 [ #include // For Visual Studio 6 in C++ mode and for many Visual Studio versions when // compiling for ARM we should wrap include with 'extern "C++" {}' // or compiler give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed #ifdef __cplusplus extern "C" { #endif # include #ifdef __cplusplus } #endif // Define _W64 macros to mark types changing their size, like intptr_t. #ifndef _W64 # if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 # define _W64 __w64 # else # define _W64 # endif #endif // 7.18.1 Integer types // 7.18.1.1 Exact-width integer types // Visual Studio 6 and Embedded Visual C++ 4 doesn't // realize that, e.g. char has the same size as __int8 // so we give up on __intX for them. #if (_MSC_VER < 1300) typedef signed char int8_t; typedef signed short int16_t; typedef signed int int32_t; typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; #else typedef signed __int8 int8_t; typedef signed __int16 int16_t; typedef signed __int32 int32_t; typedef unsigned __int8 uint8_t; typedef unsigned __int16 uint16_t; typedef unsigned __int32 uint32_t; #endif typedef signed __int64 int64_t; typedef unsigned __int64 uint64_t; // 7.18.1.2 Minimum-width integer types typedef int8_t int_least8_t; typedef int16_t int_least16_t; typedef int32_t int_least32_t; typedef int64_t int_least64_t; typedef uint8_t uint_least8_t; typedef uint16_t uint_least16_t; typedef uint32_t uint_least32_t; typedef uint64_t uint_least64_t; // 7.18.1.3 Fastest minimum-width integer types typedef int8_t int_fast8_t; typedef int16_t int_fast16_t; typedef int32_t int_fast32_t; typedef int64_t int_fast64_t; typedef uint8_t uint_fast8_t; typedef uint16_t uint_fast16_t; typedef uint32_t uint_fast32_t; typedef uint64_t uint_fast64_t; // 7.18.1.4 Integer types capable of holding object pointers #ifdef _WIN64 // [ typedef signed __int64 intptr_t; typedef unsigned __int64 uintptr_t; #else // _WIN64 ][ typedef _W64 signed int intptr_t; typedef _W64 unsigned int uintptr_t; #endif // _WIN64 ] // 7.18.1.5 Greatest-width integer types typedef int64_t intmax_t; typedef uint64_t uintmax_t; // 7.18.2 Limits of specified-width integer types #if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 // 7.18.2.1 Limits of exact-width integer types #define INT8_MIN ((int8_t)_I8_MIN) #define INT8_MAX _I8_MAX #define INT16_MIN ((int16_t)_I16_MIN) #define INT16_MAX _I16_MAX #define INT32_MIN ((int32_t)_I32_MIN) #define INT32_MAX _I32_MAX #define INT64_MIN ((int64_t)_I64_MIN) #define INT64_MAX _I64_MAX #define UINT8_MAX _UI8_MAX #define UINT16_MAX _UI16_MAX #define UINT32_MAX _UI32_MAX #define UINT64_MAX _UI64_MAX // 7.18.2.2 Limits of minimum-width integer types #define INT_LEAST8_MIN INT8_MIN #define INT_LEAST8_MAX INT8_MAX #define INT_LEAST16_MIN INT16_MIN #define INT_LEAST16_MAX INT16_MAX #define INT_LEAST32_MIN INT32_MIN #define INT_LEAST32_MAX INT32_MAX #define INT_LEAST64_MIN INT64_MIN #define INT_LEAST64_MAX INT64_MAX #define UINT_LEAST8_MAX UINT8_MAX #define UINT_LEAST16_MAX UINT16_MAX #define UINT_LEAST32_MAX UINT32_MAX #define UINT_LEAST64_MAX UINT64_MAX // 7.18.2.3 Limits of fastest minimum-width integer types #define INT_FAST8_MIN INT8_MIN #define INT_FAST8_MAX INT8_MAX #define INT_FAST16_MIN INT16_MIN #define INT_FAST16_MAX INT16_MAX #define INT_FAST32_MIN INT32_MIN #define INT_FAST32_MAX INT32_MAX #define INT_FAST64_MIN INT64_MIN #define INT_FAST64_MAX INT64_MAX #define UINT_FAST8_MAX UINT8_MAX #define UINT_FAST16_MAX UINT16_MAX #define UINT_FAST32_MAX UINT32_MAX #define UINT_FAST64_MAX UINT64_MAX // 7.18.2.4 Limits of integer types capable of holding object pointers #ifdef _WIN64 // [ # define INTPTR_MIN INT64_MIN # define INTPTR_MAX INT64_MAX # define UINTPTR_MAX UINT64_MAX #else // _WIN64 ][ # define INTPTR_MIN INT32_MIN # define INTPTR_MAX INT32_MAX # define UINTPTR_MAX UINT32_MAX #endif // _WIN64 ] // 7.18.2.5 Limits of greatest-width integer types #define INTMAX_MIN INT64_MIN #define INTMAX_MAX INT64_MAX #define UINTMAX_MAX UINT64_MAX // 7.18.3 Limits of other integer types #ifdef _WIN64 // [ # define PTRDIFF_MIN _I64_MIN # define PTRDIFF_MAX _I64_MAX #else // _WIN64 ][ # define PTRDIFF_MIN _I32_MIN # define PTRDIFF_MAX _I32_MAX #endif // _WIN64 ] #define SIG_ATOMIC_MIN INT_MIN #define SIG_ATOMIC_MAX INT_MAX #ifndef SIZE_MAX // [ # ifdef _WIN64 // [ # define SIZE_MAX _UI64_MAX # else // _WIN64 ][ # define SIZE_MAX _UI32_MAX # endif // _WIN64 ] #endif // SIZE_MAX ] // WCHAR_MIN and WCHAR_MAX are also defined in #ifndef WCHAR_MIN // [ # define WCHAR_MIN 0 #endif // WCHAR_MIN ] #ifndef WCHAR_MAX // [ # define WCHAR_MAX _UI16_MAX #endif // WCHAR_MAX ] #define WINT_MIN 0 #define WINT_MAX _UI16_MAX #endif // __STDC_LIMIT_MACROS ] // 7.18.4 Limits of other integer types #if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 // 7.18.4.1 Macros for minimum-width integer constants #define INT8_C(val) val##i8 #define INT16_C(val) val##i16 #define INT32_C(val) val##i32 #define INT64_C(val) val##i64 #define UINT8_C(val) val##ui8 #define UINT16_C(val) val##ui16 #define UINT32_C(val) val##ui32 #define UINT64_C(val) val##ui64 // 7.18.4.2 Macros for greatest-width integer constants // These #ifndef's are needed to prevent collisions with . // Check out Issue 9 for the details. #ifndef INTMAX_C // [ # define INTMAX_C INT64_C #endif // INTMAX_C ] #ifndef UINTMAX_C // [ # define UINTMAX_C UINT64_C #endif // UINTMAX_C ] #endif // __STDC_CONSTANT_MACROS ] #endif // _MSC_VER >= 1600 ] #endif // _MSC_STDINT_H_ ] pymongo-3.6.1/bson/time64.h0000644000076600000240000000275113245621354015667 0ustar shanestaff00000000000000#ifndef TIME64_H # define TIME64_H #include #include "time64_config.h" /* Set our custom types */ typedef INT_64_T Int64; typedef Int64 Time64_T; typedef Int64 Year; /* A copy of the tm struct but with a 64 bit year */ struct TM64 { int tm_sec; int tm_min; int tm_hour; int tm_mday; int tm_mon; Year tm_year; int tm_wday; int tm_yday; int tm_isdst; #ifdef HAS_TM_TM_GMTOFF long tm_gmtoff; #endif #ifdef HAS_TM_TM_ZONE char *tm_zone; #endif }; /* Decide which tm struct to use */ #ifdef USE_TM64 #define TM TM64 #else #define TM tm #endif /* Declare public functions */ struct TM *gmtime64_r (const Time64_T *, struct TM *); struct TM *localtime64_r (const Time64_T *, struct TM *); struct TM *gmtime64 (const Time64_T *); struct TM *localtime64 (const Time64_T *); Time64_T timegm64 (const struct TM *); Time64_T mktime64 (const struct TM *); Time64_T timelocal64 (const struct TM *); /* Not everyone has gm/localtime_r(), provide a replacement */ #ifdef HAS_LOCALTIME_R # define LOCALTIME_R(clock, result) localtime_r(clock, result) #else # define LOCALTIME_R(clock, result) fake_localtime_r(clock, result) #endif #ifdef HAS_GMTIME_R # define GMTIME_R(clock, result) gmtime_r(clock, result) #else # define GMTIME_R(clock, result) fake_gmtime_r(clock, result) #endif #endif pymongo-3.6.1/bson/max_key.py0000644000076600000240000000244313245621354016413 0ustar shanestaff00000000000000# Copyright 2010-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Representation for the MongoDB internal MaxKey type. """ class MaxKey(object): """MongoDB internal MaxKey type. .. versionchanged:: 2.7 ``MaxKey`` now implements comparison operators. """ _type_marker = 127 def __eq__(self, other): return isinstance(other, MaxKey) def __hash__(self): return hash(self._type_marker) def __ne__(self, other): return not self == other def __le__(self, other): return isinstance(other, MaxKey) def __lt__(self, dummy): return False def __ge__(self, dummy): return True def __gt__(self, other): return not isinstance(other, MaxKey) def __repr__(self): return "MaxKey()" pymongo-3.6.1/bson/errors.py0000644000076600000240000000220713245621354016270 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Exceptions raised by the BSON package.""" class BSONError(Exception): """Base class for all BSON exceptions. """ class InvalidBSON(BSONError): """Raised when trying to create a BSON object from invalid data. """ class InvalidStringData(BSONError): """Raised when trying to encode a string containing non-UTF8 data. """ class InvalidDocument(BSONError): """Raised when trying to create a BSON object from an invalid document. """ class InvalidId(BSONError): """Raised when trying to create an ObjectId from invalid data. """ pymongo-3.6.1/bson/timestamp.py0000644000076600000240000000753413245617773017001 0ustar shanestaff00000000000000# Copyright 2010-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing MongoDB internal Timestamps. """ import calendar import datetime from bson.py3compat import integer_types from bson.tz_util import utc UPPERBOUND = 4294967296 class Timestamp(object): """MongoDB internal timestamps used in the opLog. """ _type_marker = 17 def __init__(self, time, inc): """Create a new :class:`Timestamp`. This class is only for use with the MongoDB opLog. If you need to store a regular timestamp, please use a :class:`~datetime.datetime`. Raises :class:`TypeError` if `time` is not an instance of :class: `int` or :class:`~datetime.datetime`, or `inc` is not an instance of :class:`int`. Raises :class:`ValueError` if `time` or `inc` is not in [0, 2**32). :Parameters: - `time`: time in seconds since epoch UTC, or a naive UTC :class:`~datetime.datetime`, or an aware :class:`~datetime.datetime` - `inc`: the incrementing counter """ if isinstance(time, datetime.datetime): if time.utcoffset() is not None: time = time - time.utcoffset() time = int(calendar.timegm(time.timetuple())) if not isinstance(time, integer_types): raise TypeError("time must be an instance of int") if not isinstance(inc, integer_types): raise TypeError("inc must be an instance of int") if not 0 <= time < UPPERBOUND: raise ValueError("time must be contained in [0, 2**32)") if not 0 <= inc < UPPERBOUND: raise ValueError("inc must be contained in [0, 2**32)") self.__time = time self.__inc = inc @property def time(self): """Get the time portion of this :class:`Timestamp`. """ return self.__time @property def inc(self): """Get the inc portion of this :class:`Timestamp`. """ return self.__inc def __eq__(self, other): if isinstance(other, Timestamp): return (self.__time == other.time and self.__inc == other.inc) else: return NotImplemented def __hash__(self): return hash(self.time) ^ hash(self.inc) def __ne__(self, other): return not self == other def __lt__(self, other): if isinstance(other, Timestamp): return (self.time, self.inc) < (other.time, other.inc) return NotImplemented def __le__(self, other): if isinstance(other, Timestamp): return (self.time, self.inc) <= (other.time, other.inc) return NotImplemented def __gt__(self, other): if isinstance(other, Timestamp): return (self.time, self.inc) > (other.time, other.inc) return NotImplemented def __ge__(self, other): if isinstance(other, Timestamp): return (self.time, self.inc) >= (other.time, other.inc) return NotImplemented def __repr__(self): return "Timestamp(%s, %s)" % (self.__time, self.__inc) def as_datetime(self): """Return a :class:`~datetime.datetime` instance corresponding to the time portion of this :class:`Timestamp`. The returned datetime's timezone is UTC. """ return datetime.datetime.fromtimestamp(self.__time, utc) pymongo-3.6.1/bson/son.py0000644000076600000240000001323413245621354015555 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for creating and manipulating SON, the Serialized Ocument Notation. Regular dictionaries can be used instead of SON objects, but not when the order of keys is important. A SON object can be used just like a normal Python dictionary.""" import copy import re from bson.py3compat import abc, iteritems # This sort of sucks, but seems to be as good as it gets... # This is essentially the same as re._pattern_type RE_TYPE = type(re.compile("")) class SON(dict): """SON data. A subclass of dict that maintains ordering of keys and provides a few extra niceties for dealing with SON. SON provides an API similar to collections.OrderedDict from Python 2.7+. """ def __init__(self, data=None, **kwargs): self.__keys = [] dict.__init__(self) self.update(data) self.update(kwargs) def __new__(cls, *args, **kwargs): instance = super(SON, cls).__new__(cls, *args, **kwargs) instance.__keys = [] return instance def __repr__(self): result = [] for key in self.__keys: result.append("(%r, %r)" % (key, self[key])) return "SON([%s])" % ", ".join(result) def __setitem__(self, key, value): if key not in self.__keys: self.__keys.append(key) dict.__setitem__(self, key, value) def __delitem__(self, key): self.__keys.remove(key) dict.__delitem__(self, key) def keys(self): return list(self.__keys) def copy(self): other = SON() other.update(self) return other # TODO this is all from UserDict.DictMixin. it could probably be made more # efficient. # second level definitions support higher levels def __iter__(self): for k in self.__keys: yield k def has_key(self, key): return key in self.__keys # third level takes advantage of second level definitions def iteritems(self): for k in self: yield (k, self[k]) def iterkeys(self): return self.__iter__() # fourth level uses definitions from lower levels def itervalues(self): for _, v in self.iteritems(): yield v def values(self): return [v for _, v in self.iteritems()] def items(self): return [(key, self[key]) for key in self] def clear(self): self.__keys = [] super(SON, self).clear() def setdefault(self, key, default=None): try: return self[key] except KeyError: self[key] = default return default def pop(self, key, *args): if len(args) > 1: raise TypeError("pop expected at most 2 arguments, got "\ + repr(1 + len(args))) try: value = self[key] except KeyError: if args: return args[0] raise del self[key] return value def popitem(self): try: k, v = next(self.iteritems()) except StopIteration: raise KeyError('container is empty') del self[k] return (k, v) def update(self, other=None, **kwargs): # Make progressively weaker assumptions about "other" if other is None: pass elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups for k, v in other.iteritems(): self[k] = v elif hasattr(other, 'keys'): for k in other.keys(): self[k] = other[k] else: for k, v in other: self[k] = v if kwargs: self.update(kwargs) def get(self, key, default=None): try: return self[key] except KeyError: return default def __eq__(self, other): """Comparison to another SON is order-sensitive while comparison to a regular dictionary is order-insensitive. """ if isinstance(other, SON): return len(self) == len(other) and self.items() == other.items() return self.to_dict() == other def __ne__(self, other): return not self == other def __len__(self): return len(self.__keys) def to_dict(self): """Convert a SON document to a normal Python dictionary instance. This is trickier than just *dict(...)* because it needs to be recursive. """ def transform_value(value): if isinstance(value, list): return [transform_value(v) for v in value] elif isinstance(value, abc.Mapping): return dict([ (k, transform_value(v)) for k, v in iteritems(value)]) else: return value return transform_value(dict(self)) def __deepcopy__(self, memo): out = SON() val_id = id(self) if val_id in memo: return memo.get(val_id) memo[val_id] = out for k, v in self.iteritems(): if not isinstance(v, RE_TYPE): v = copy.deepcopy(v, memo) out[k] = v return out pymongo-3.6.1/bson/min_key.py0000644000076600000240000000244313245621354016411 0ustar shanestaff00000000000000# Copyright 2010-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Representation for the MongoDB internal MinKey type. """ class MinKey(object): """MongoDB internal MinKey type. .. versionchanged:: 2.7 ``MinKey`` now implements comparison operators. """ _type_marker = 255 def __eq__(self, other): return isinstance(other, MinKey) def __hash__(self): return hash(self._type_marker) def __ne__(self, other): return not self == other def __le__(self, dummy): return True def __lt__(self, other): return not isinstance(other, MinKey) def __ge__(self, other): return isinstance(other, MinKey) def __gt__(self, dummy): return False def __repr__(self): return "MinKey()" pymongo-3.6.1/bson/codec_options.py0000644000076600000240000001443113245621354017606 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for specifying BSON codec options.""" import datetime from collections import namedtuple from bson.py3compat import abc, string_type from bson.binary import (ALL_UUID_REPRESENTATIONS, PYTHON_LEGACY, UUID_REPRESENTATION_NAMES) _RAW_BSON_DOCUMENT_MARKER = 101 def _raw_document_class(document_class): """Determine if a document_class is a RawBSONDocument class.""" marker = getattr(document_class, '_type_marker', None) return marker == _RAW_BSON_DOCUMENT_MARKER _options_base = namedtuple( 'CodecOptions', ('document_class', 'tz_aware', 'uuid_representation', 'unicode_decode_error_handler', 'tzinfo')) class CodecOptions(_options_base): """Encapsulates BSON options used in CRUD operations. :Parameters: - `document_class`: BSON documents returned in queries will be decoded to an instance of this class. Must be a subclass of :class:`~collections.MutableMapping`. Defaults to :class:`dict`. - `tz_aware`: If ``True``, BSON datetimes will be decoded to timezone aware instances of :class:`~datetime.datetime`. Otherwise they will be naive. Defaults to ``False``. - `uuid_representation`: The BSON representation to use when encoding and decoding instances of :class:`~uuid.UUID`. Defaults to :data:`~bson.binary.PYTHON_LEGACY`. - `unicode_decode_error_handler`: The error handler to use when decoding an invalid BSON string. Valid options include 'strict', 'replace', and 'ignore'. Defaults to 'strict'. - `tzinfo`: A :class:`~datetime.tzinfo` subclass that specifies the timezone to/from which :class:`~datetime.datetime` objects should be encoded/decoded. .. warning:: Care must be taken when changing `unicode_decode_error_handler` from its default value ('strict'). The 'replace' and 'ignore' modes should not be used when documents retrieved from the server will be modified in the client application and stored back to the server. """ def __new__(cls, document_class=dict, tz_aware=False, uuid_representation=PYTHON_LEGACY, unicode_decode_error_handler="strict", tzinfo=None): if not (issubclass(document_class, abc.MutableMapping) or _raw_document_class(document_class)): raise TypeError("document_class must be dict, bson.son.SON, " "bson.raw_bson.RawBSONDocument, or a " "sublass of collections.MutableMapping") if not isinstance(tz_aware, bool): raise TypeError("tz_aware must be True or False") if uuid_representation not in ALL_UUID_REPRESENTATIONS: raise ValueError("uuid_representation must be a value " "from bson.binary.ALL_UUID_REPRESENTATIONS") if not isinstance(unicode_decode_error_handler, (string_type, None)): raise ValueError("unicode_decode_error_handler must be a string " "or None") if tzinfo is not None: if not isinstance(tzinfo, datetime.tzinfo): raise TypeError( "tzinfo must be an instance of datetime.tzinfo") if not tz_aware: raise ValueError( "cannot specify tzinfo without also setting tz_aware=True") return tuple.__new__( cls, (document_class, tz_aware, uuid_representation, unicode_decode_error_handler, tzinfo)) def _arguments_repr(self): """Representation of the arguments used to create this object.""" document_class_repr = ( 'dict' if self.document_class is dict else repr(self.document_class)) uuid_rep_repr = UUID_REPRESENTATION_NAMES.get(self.uuid_representation, self.uuid_representation) return ('document_class=%s, tz_aware=%r, uuid_representation=' '%s, unicode_decode_error_handler=%r, tzinfo=%r' % (document_class_repr, self.tz_aware, uuid_rep_repr, self.unicode_decode_error_handler, self.tzinfo)) def __repr__(self): return '%s(%s)' % (self.__class__.__name__, self._arguments_repr()) def with_options(self, **kwargs): """Make a copy of this CodecOptions, overriding some options:: >>> from bson.codec_options import DEFAULT_CODEC_OPTIONS >>> DEFAULT_CODEC_OPTIONS.tz_aware False >>> options = DEFAULT_CODEC_OPTIONS.with_options(tz_aware=True) >>> options.tz_aware True .. versionadded:: 3.5 """ return CodecOptions( kwargs.get('document_class', self.document_class), kwargs.get('tz_aware', self.tz_aware), kwargs.get('uuid_representation', self.uuid_representation), kwargs.get('unicode_decode_error_handler', self.unicode_decode_error_handler), kwargs.get('tzinfo', self.tzinfo)) DEFAULT_CODEC_OPTIONS = CodecOptions() def _parse_codec_options(options): """Parse BSON codec options.""" return CodecOptions( document_class=options.get( 'document_class', DEFAULT_CODEC_OPTIONS.document_class), tz_aware=options.get( 'tz_aware', DEFAULT_CODEC_OPTIONS.tz_aware), uuid_representation=options.get( 'uuidrepresentation', DEFAULT_CODEC_OPTIONS.uuid_representation), unicode_decode_error_handler=options.get( 'unicode_decode_error_handler', DEFAULT_CODEC_OPTIONS.unicode_decode_error_handler), tzinfo=options.get('tzinfo', DEFAULT_CODEC_OPTIONS.tzinfo)) pymongo-3.6.1/bson/regex.py0000644000076600000240000001030313245621354016062 0ustar shanestaff00000000000000# Copyright 2013-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing MongoDB regular expressions. """ import re from bson.son import RE_TYPE from bson.py3compat import string_type, text_type def str_flags_to_int(str_flags): flags = 0 if "i" in str_flags: flags |= re.IGNORECASE if "l" in str_flags: flags |= re.LOCALE if "m" in str_flags: flags |= re.MULTILINE if "s" in str_flags: flags |= re.DOTALL if "u" in str_flags: flags |= re.UNICODE if "x" in str_flags: flags |= re.VERBOSE return flags class Regex(object): """BSON regular expression data.""" _type_marker = 11 @classmethod def from_native(cls, regex): """Convert a Python regular expression into a ``Regex`` instance. Note that in Python 3, a regular expression compiled from a :class:`str` has the ``re.UNICODE`` flag set. If it is undesirable to store this flag in a BSON regular expression, unset it first:: >>> pattern = re.compile('.*') >>> regex = Regex.from_native(pattern) >>> regex.flags ^= re.UNICODE >>> db.collection.insert({'pattern': regex}) :Parameters: - `regex`: A regular expression object from ``re.compile()``. .. warning:: Python regular expressions use a different syntax and different set of flags than MongoDB, which uses `PCRE`_. A regular expression retrieved from the server may not compile in Python, or may match a different set of strings in Python than when used in a MongoDB query. .. _PCRE: http://www.pcre.org/ """ if not isinstance(regex, RE_TYPE): raise TypeError( "regex must be a compiled regular expression, not %s" % type(regex)) return Regex(regex.pattern, regex.flags) def __init__(self, pattern, flags=0): """BSON regular expression data. This class is useful to store and retrieve regular expressions that are incompatible with Python's regular expression dialect. :Parameters: - `pattern`: string - `flags`: (optional) an integer bitmask, or a string of flag characters like "im" for IGNORECASE and MULTILINE """ if not isinstance(pattern, (text_type, bytes)): raise TypeError("pattern must be a string, not %s" % type(pattern)) self.pattern = pattern if isinstance(flags, string_type): self.flags = str_flags_to_int(flags) elif isinstance(flags, int): self.flags = flags else: raise TypeError( "flags must be a string or int, not %s" % type(flags)) def __eq__(self, other): if isinstance(other, Regex): return self.pattern == other.pattern and self.flags == other.flags else: return NotImplemented __hash__ = None def __ne__(self, other): return not self == other def __repr__(self): return "Regex(%r, %r)" % (self.pattern, self.flags) def try_compile(self): """Compile this :class:`Regex` as a Python regular expression. .. warning:: Python regular expressions use a different syntax and different set of flags than MongoDB, which uses `PCRE`_. A regular expression retrieved from the server may not compile in Python, or may match a different set of strings in Python than when used in a MongoDB query. :meth:`try_compile()` may raise :exc:`re.error`. .. _PCRE: http://www.pcre.org/ """ return re.compile(self.pattern, self.flags) pymongo-3.6.1/PKG-INFO0000644000076600000240000002201513246104133014525 0ustar shanestaff00000000000000Metadata-Version: 1.1 Name: pymongo Version: 3.6.1 Summary: Python driver for MongoDB Home-page: http://github.com/mongodb/mongo-python-driver Author: Bernie Hackett Author-email: bernie@mongodb.com License: Apache License, Version 2.0 Description: ======= PyMongo ======= :Info: See `the mongo site `_ for more information. See `github `_ for the latest source. :Author: Mike Dirolf :Maintainer: Bernie Hackett About ===== The PyMongo distribution contains tools for interacting with MongoDB database from Python. The ``bson`` package is an implementation of the `BSON format `_ for Python. The ``pymongo`` package is a native Python driver for MongoDB. The ``gridfs`` package is a `gridfs `_ implementation on top of ``pymongo``. PyMongo supports MongoDB 2.6, 3.0, 3.2, 3.4, and 3.6. Support / Feedback ================== For issues with, questions about, or feedback for PyMongo, please look into our `support channels `_. Please do not email any of the PyMongo developers directly with issues or questions - you're more likely to get an answer on the `mongodb-user `_ list on Google Groups. Bugs / Feature Requests ======================= Think you’ve found a bug? Want to see a new feature in PyMongo? Please open a case in our issue management tool, JIRA: - `Create an account and login `_. - Navigate to `the PYTHON project `_. - Click **Create Issue** - Please provide as much information as possible about the issue type and how to reproduce it. Bug reports in JIRA for all driver projects (i.e. PYTHON, CSHARP, JAVA) and the Core Server (i.e. SERVER) project are **public**. How To Ask For Help ------------------- Please include all of the following information when opening an issue: - Detailed steps to reproduce the problem, including full traceback, if possible. - The exact python version used, with patch level:: $ python -c "import sys; print(sys.version)" - The exact version of PyMongo used, with patch level:: $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())" - The operating system and version (e.g. Windows 7, OSX 10.8, ...) - Web framework or asynchronous network library used, if any, with version (e.g. Django 1.7, mod_wsgi 4.3.0, gevent 1.0.1, Tornado 4.0.2, ...) Security Vulnerabilities ------------------------ If you’ve identified a security vulnerability in a driver or any other MongoDB project, please report it according to the `instructions here `_. Installation ============ PyMongo can be installed with `pip `_:: $ python -m pip install pymongo Or ``easy_install`` from `setuptools `_:: $ python -m easy_install pymongo You can also download the project source and do:: $ python setup.py install Do **not** install the "bson" package from pypi. PyMongo comes with its own bson package; doing "easy_install bson" installs a third-party package that is incompatible with PyMongo. Dependencies ============ PyMongo supports CPython 2.6, 2.7, 3.4+, PyPy, and PyPy3. Optional dependencies: GSSAPI authentication requires `pykerberos `_ on Unix or `WinKerberos `_ on Windows. The correct dependency can be installed automatically along with PyMongo:: $ python -m pip install pymongo[gssapi] Support for mongodb+srv:// URIs requires `dnspython `_:: $ python -m pip install pymongo[srv] TLS / SSL support may require `ipaddress `_ and `certifi `_ or `wincertstore `_ depending on the Python version in use. The necessary dependencies can be installed along with PyMongo:: $ python -m pip install pymongo[tls] You can install all dependencies automatically with the following command:: $ python -m pip install pymongo[gssapi,srv,tls] Other optional packages: - `backports.pbkdf2 `_, improves authentication performance with SCRAM-SHA-1, the default authentication mechanism for MongoDB 3.0+. It especially improves performance on Python versions older than 2.7.8. - `monotonic `_ adds support for a monotonic clock, which improves reliability in environments where clock adjustments are frequent. Not needed in Python 3. Additional dependencies are: - (to generate documentation) sphinx_ - (to run the tests under Python 2.6) unittest2_ Examples ======== Here's a basic example (for more see the *examples* section of the docs): .. code-block:: python >>> import pymongo >>> client = pymongo.MongoClient("localhost", 27017) >>> db = client.test >>> db.name u'test' >>> db.my_collection Collection(Database(MongoClient('localhost', 27017), u'test'), u'my_collection') >>> db.my_collection.insert_one({"x": 10}).inserted_id ObjectId('4aba15ebe23f6b53b0000000') >>> db.my_collection.insert_one({"x": 8}).inserted_id ObjectId('4aba160ee23f6b543e000000') >>> db.my_collection.insert_one({"x": 11}).inserted_id ObjectId('4aba160ee23f6b543e000002') >>> db.my_collection.find_one() {u'x': 10, u'_id': ObjectId('4aba15ebe23f6b53b0000000')} >>> for item in db.my_collection.find(): ... print(item["x"]) ... 10 8 11 >>> db.my_collection.create_index("x") u'x_1' >>> for item in db.my_collection.find().sort("x", pymongo.ASCENDING): ... print(item["x"]) ... 8 10 11 >>> [item["x"] for item in db.my_collection.find().limit(2).skip(1)] [8, 11] Documentation ============= You will need sphinx_ installed to generate the documentation. Documentation can be generated by running **python setup.py doc**. Generated documentation can be found in the *doc/build/html/* directory. Testing ======= The easiest way to run the tests is to run **python setup.py test** in the root of the distribution. Note that you will need unittest2_ to run the tests under Python 2.6. To verify that PyMongo works with Gevent's monkey-patching:: $ python green_framework_test.py gevent Or with Eventlet's:: $ python green_framework_test.py eventlet .. _sphinx: http://sphinx.pocoo.org/ .. _unittest2: https://pypi.python.org/pypi/unittest2 Keywords: mongo,mongodb,pymongo,gridfs,bson Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Apache Software License Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: POSIX Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Database pymongo-3.6.1/tools/0000755000076600000240000000000013246104133014570 5ustar shanestaff00000000000000pymongo-3.6.1/tools/clean.py0000644000076600000240000000213013156613521016226 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Clean up script for build artifacts. Only really intended to be used by internal build scripts. """ import os import sys try: os.remove("pymongo/_cmessage.so") os.remove("bson/_cbson.so") except: pass try: os.remove("pymongo/_cmessage.pyd") os.remove("bson/_cbson.pyd") except: pass try: from pymongo import _cmessage sys.exit("could still import _cmessage") except ImportError: pass try: from bson import _cbson sys.exit("could still import _cbson") except ImportError: pass pymongo-3.6.1/tools/benchmark.py0000644000076600000240000001346013245617773017122 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """MongoDB benchmarking suite.""" import time import sys sys.path[0:0] = [""] import datetime from pymongo import mongo_client from pymongo import ASCENDING trials = 2 per_trial = 5000 batch_size = 100 small = {} medium = {"integer": 5, "number": 5.05, "boolean": False, "array": ["test", "benchmark"] } # this is similar to the benchmark data posted to the user list large = {"base_url": "http://www.example.com/test-me", "total_word_count": 6743, "access_time": datetime.datetime.utcnow(), "meta_tags": {"description": "i am a long description string", "author": "Holly Man", "dynamically_created_meta_tag": "who know\n what" }, "page_structure": {"counted_tags": 3450, "no_of_js_attached": 10, "no_of_images": 6 }, "harvested_words": ["10gen", "web", "open", "source", "application", "paas", "platform-as-a-service", "technology", "helps", "developers", "focus", "building", "mongodb", "mongo"] * 20 } def setup_insert(db, collection, object): db.drop_collection(collection) def insert(db, collection, object): for i in range(per_trial): to_insert = object.copy() to_insert["x"] = i db[collection].insert(to_insert) def insert_batch(db, collection, object): for i in range(per_trial / batch_size): db[collection].insert([object] * batch_size) def find_one(db, collection, x): for _ in range(per_trial): db[collection].find_one({"x": x}) def find(db, collection, x): for _ in range(per_trial): for _ in db[collection].find({"x": x}): pass def timed(name, function, args=[], setup=None): times = [] for _ in range(trials): if setup: setup(*args) start = time.time() function(*args) times.append(time.time() - start) best_time = min(times) print "%s%d" % (name + (60 - len(name)) * ".", per_trial / best_time) return best_time def main(): c = mongo_client.MongoClient(connectTimeoutMS=60*1000) # jack up timeout c.drop_database("benchmark") db = c.benchmark timed("insert (small, no index)", insert, [db, 'small_none', small], setup_insert) timed("insert (medium, no index)", insert, [db, 'medium_none', medium], setup_insert) timed("insert (large, no index)", insert, [db, 'large_none', large], setup_insert) db.small_index.create_index("x", ASCENDING) timed("insert (small, indexed)", insert, [db, 'small_index', small]) db.medium_index.create_index("x", ASCENDING) timed("insert (medium, indexed)", insert, [db, 'medium_index', medium]) db.large_index.create_index("x", ASCENDING) timed("insert (large, indexed)", insert, [db, 'large_index', large]) timed("batch insert (small, no index)", insert_batch, [db, 'small_bulk', small], setup_insert) timed("batch insert (medium, no index)", insert_batch, [db, 'medium_bulk', medium], setup_insert) timed("batch insert (large, no index)", insert_batch, [db, 'large_bulk', large], setup_insert) timed("find_one (small, no index)", find_one, [db, 'small_none', per_trial / 2]) timed("find_one (medium, no index)", find_one, [db, 'medium_none', per_trial / 2]) timed("find_one (large, no index)", find_one, [db, 'large_none', per_trial / 2]) timed("find_one (small, indexed)", find_one, [db, 'small_index', per_trial / 2]) timed("find_one (medium, indexed)", find_one, [db, 'medium_index', per_trial / 2]) timed("find_one (large, indexed)", find_one, [db, 'large_index', per_trial / 2]) timed("find (small, no index)", find, [db, 'small_none', per_trial / 2]) timed("find (medium, no index)", find, [db, 'medium_none', per_trial / 2]) timed("find (large, no index)", find, [db, 'large_none', per_trial / 2]) timed("find (small, indexed)", find, [db, 'small_index', per_trial / 2]) timed("find (medium, indexed)", find, [db, 'medium_index', per_trial / 2]) timed("find (large, indexed)", find, [db, 'large_index', per_trial / 2]) # timed("find range (small, no index)", find, # [db, 'small_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) # timed("find range (medium, no index)", find, # [db, 'medium_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) # timed("find range (large, no index)", find, # [db, 'large_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) timed("find range (small, indexed)", find, [db, 'small_index', {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) timed("find range (medium, indexed)", find, [db, 'medium_index', {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) timed("find range (large, indexed)", find, [db, 'large_index', {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) if __name__ == "__main__": # cProfile.run("main()") main() pymongo-3.6.1/tools/fail_if_no_c.py0000644000076600000240000000151213156613521017536 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Fail if the C extension module doesn't exist. Only really intended to be used by internal build scripts. """ import sys sys.path[0:0] = [""] import bson import pymongo if not pymongo.has_c() or not bson.has_c(): sys.exit("could not load C extensions") pymongo-3.6.1/tools/README.rst0000644000076600000240000000011712727574422016275 0ustar shanestaff00000000000000Tools ===== This directory contains tools for use with the ``pymongo`` module. pymongo-3.6.1/LICENSE0000644000076600000240000002613513245621354014454 0ustar shanestaff00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pymongo-3.6.1/test/0000755000076600000240000000000013246104133014407 5ustar shanestaff00000000000000pymongo-3.6.1/test/test_read_preferences.py0000644000076600000240000006023613245621354021333 0ustar shanestaff00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the replica_set_connection module.""" import contextlib import copy import pickle import random import sys import warnings sys.path[0:0] = [""] from bson.py3compat import MAXSIZE from bson.son import SON from pymongo.errors import ConfigurationError, OperationFailure from pymongo.message import _maybe_add_read_preference from pymongo.mongo_client import MongoClient from pymongo.read_preferences import (ReadPreference, MovingAverage, Primary, PrimaryPreferred, Secondary, SecondaryPreferred, Nearest) from pymongo.server_description import ServerDescription from pymongo.server_selectors import readable_server_selector, Selection from pymongo.server_type import SERVER_TYPE from pymongo.write_concern import WriteConcern from test.test_replica_set_client import TestReplicaSetClientBase from test import (SkipTest, client_context, unittest, db_user, db_pwd) from test.utils import connected, single_client, one, wait_until, rs_client from test.version import Version class TestSelections(unittest.TestCase): @client_context.require_connection def test_bool(self): client = single_client() wait_until(lambda: client.address, "discover primary") selection = Selection.from_topology_description( client._topology.description) self.assertTrue(selection) self.assertFalse(selection.with_server_descriptions([])) class TestReadPreferenceObjects(unittest.TestCase): prefs = [Primary(), PrimaryPreferred(), Secondary(), Nearest(tag_sets=[{'a': 1}, {'b': 2}]), SecondaryPreferred(max_staleness=30)] def test_pickle(self): for pref in self.prefs: self.assertEqual(pref, pickle.loads(pickle.dumps(pref))) def test_copy(self): for pref in self.prefs: self.assertEqual(pref, copy.copy(pref)) def test_deepcopy(self): for pref in self.prefs: self.assertEqual(pref, copy.deepcopy(pref)) class TestReadPreferencesBase(TestReplicaSetClientBase): @classmethod @client_context.require_secondaries_count(1) def setUpClass(cls): super(TestReadPreferencesBase, cls).setUpClass() def setUp(self): super(TestReadPreferencesBase, self).setUp() # Insert some data so we can use cursors in read_from_which_host self.client.pymongo_test.test.drop() self.client.get_database( "pymongo_test", write_concern=WriteConcern(w=self.w)).test.insert_many( [{'_id': i} for i in range(10)]) self.addCleanup(self.client.pymongo_test.test.drop) def read_from_which_host(self, client): """Do a find() on the client and return which host was used """ cursor = client.pymongo_test.test.find() next(cursor) return cursor.address def read_from_which_kind(self, client): """Do a find() on the client and return 'primary' or 'secondary' depending on which the client used. """ address = self.read_from_which_host(client) if address == client.primary: return 'primary' elif address in client.secondaries: return 'secondary' else: self.fail( 'Cursor used address %s, expected either primary ' '%s or secondaries %s' % ( address, client.primary, client.secondaries)) def assertReadsFrom(self, expected, **kwargs): c = rs_client(**kwargs) wait_until( lambda: len(c.nodes - c.arbiters) == self.w, "discovered all nodes") used = self.read_from_which_kind(c) self.assertEqual(expected, used, 'Cursor used %s, expected %s' % ( used, expected)) class TestSingleSlaveOk(TestReadPreferencesBase): def test_reads_from_secondary(self): host, port = next(iter(self.client.secondaries)) # Direct connection to a secondary. client = single_client(host, port) self.assertFalse(client.is_primary) # Regardless of read preference, we should be able to do # "reads" with a direct connection to a secondary. # See server-selection.rst#topology-type-single. self.assertEqual(client.read_preference, ReadPreference.PRIMARY) db = client.pymongo_test coll = db.test # Test find and find_one. self.assertIsNotNone(coll.find_one()) self.assertEqual(10, len(list(coll.find()))) # Test some database helpers. self.assertIsNotNone(db.collection_names()) self.assertIsNotNone(db.validate_collection("test")) self.assertIsNotNone(db.command("count", "test")) # Test some collection helpers. self.assertEqual(10, coll.count()) self.assertEqual(10, len(coll.distinct("_id"))) self.assertIsNotNone(coll.aggregate([])) self.assertIsNotNone(coll.index_information()) # Test some "magic" namespace helpers. self.assertIsNotNone(db.current_op()) class TestReadPreferences(TestReadPreferencesBase): def test_mode_validation(self): for mode in (ReadPreference.PRIMARY, ReadPreference.PRIMARY_PREFERRED, ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED, ReadPreference.NEAREST): self.assertEqual( mode, rs_client(read_preference=mode).read_preference) self.assertRaises( TypeError, rs_client, read_preference='foo') def test_tag_sets_validation(self): S = Secondary(tag_sets=[{}]) self.assertEqual( [{}], rs_client(read_preference=S).read_preference.tag_sets) S = Secondary(tag_sets=[{'k': 'v'}]) self.assertEqual( [{'k': 'v'}], rs_client(read_preference=S).read_preference.tag_sets) S = Secondary(tag_sets=[{'k': 'v'}, {}]) self.assertEqual( [{'k': 'v'}, {}], rs_client(read_preference=S).read_preference.tag_sets) self.assertRaises(ValueError, Secondary, tag_sets=[]) # One dict not ok, must be a list of dicts self.assertRaises(TypeError, Secondary, tag_sets={'k': 'v'}) self.assertRaises(TypeError, Secondary, tag_sets='foo') self.assertRaises(TypeError, Secondary, tag_sets=['foo']) def test_threshold_validation(self): self.assertEqual(17, rs_client( localThresholdMS=17 ).local_threshold_ms) self.assertEqual(42, rs_client( localThresholdMS=42 ).local_threshold_ms) self.assertEqual(666, rs_client( localthresholdms=666 ).local_threshold_ms) self.assertEqual(0, rs_client( localthresholdms=0 ).local_threshold_ms) self.assertRaises(ValueError, rs_client, localthresholdms=-1) def test_zero_latency(self): if (client_context.version >= (3, 7, 2) and client_context.auth_enabled and client_context.is_rs): raise SkipTest("Disabled due to SERVER-32845") ping_times = set() # Generate unique ping times. while len(ping_times) < len(self.client.nodes): ping_times.add(random.random()) for ping_time, host in zip(ping_times, self.client.nodes): ServerDescription._host_to_round_trip_time[host] = ping_time try: client = connected( rs_client(readPreference='nearest', localThresholdMS=0)) wait_until( lambda: client.nodes == self.client.nodes, "discovered all nodes") host = self.read_from_which_host(client) for _ in range(5): self.assertEqual(host, self.read_from_which_host(client)) finally: ServerDescription._host_to_round_trip_time.clear() def test_primary(self): self.assertReadsFrom( 'primary', read_preference=ReadPreference.PRIMARY) def test_primary_with_tags(self): # Tags not allowed with PRIMARY self.assertRaises( ConfigurationError, rs_client, tag_sets=[{'dc': 'ny'}]) def test_primary_preferred(self): self.assertReadsFrom( 'primary', read_preference=ReadPreference.PRIMARY_PREFERRED) def test_secondary(self): self.assertReadsFrom( 'secondary', read_preference=ReadPreference.SECONDARY) def test_secondary_preferred(self): self.assertReadsFrom( 'secondary', read_preference=ReadPreference.SECONDARY_PREFERRED) def test_nearest(self): # With high localThresholdMS, expect to read from any # member c = rs_client( read_preference=ReadPreference.NEAREST, localThresholdMS=10000) # 10 seconds data_members = set(self.hosts).difference(set(self.arbiters)) # This is a probabilistic test; track which members we've read from so # far, and keep reading until we've used all the members or give up. # Chance of using only 2 of 3 members 10k times if there's no bug = # 3 * (2/3)**10000, very low. used = set() i = 0 while data_members.difference(used) and i < 10000: address = self.read_from_which_host(c) used.add(address) i += 1 not_used = data_members.difference(used) latencies = ', '.join( '%s: %dms' % (server.description.address, server.description.round_trip_time) for server in c._get_topology().select_servers( readable_server_selector)) self.assertFalse( not_used, "Expected to use primary and all secondaries for mode NEAREST," " but didn't use %s\nlatencies: %s" % (not_used, latencies)) class ReadPrefTester(MongoClient): def __init__(self, *args, **kwargs): self.has_read_from = set() client_options = client_context.ssl_client_options.copy() client_options.update(kwargs) super(ReadPrefTester, self).__init__(*args, **client_options) @contextlib.contextmanager def _socket_for_reads(self, read_preference): context = super(ReadPrefTester, self)._socket_for_reads(read_preference) with context as (sock_info, slave_ok): self.record_a_read(sock_info.address) yield sock_info, slave_ok def record_a_read(self, address): server = self._get_topology().select_server_by_address(address, 0) self.has_read_from.add(server) _PREF_MAP = [ (Primary, SERVER_TYPE.RSPrimary), (PrimaryPreferred, SERVER_TYPE.RSPrimary), (Secondary, SERVER_TYPE.RSSecondary), (SecondaryPreferred, SERVER_TYPE.RSSecondary), (Nearest, 'any') ] class TestCommandAndReadPreference(TestReplicaSetClientBase): @classmethod @client_context.require_secondaries_count(1) def setUpClass(cls): super(TestCommandAndReadPreference, cls).setUpClass() cls.c = ReadPrefTester( client_context.pair, replicaSet=cls.name, # Ignore round trip times, to test ReadPreference modes only. localThresholdMS=1000*1000) if client_context.auth_enabled: cls.c.admin.authenticate(db_user, db_pwd) cls.client_version = Version.from_client(cls.c) # mapReduce and group fail with no collection coll = cls.c.pymongo_test.get_collection( 'test', write_concern=WriteConcern(w=cls.w)) coll.insert_one({}) @classmethod def tearDownClass(cls): cls.c.drop_database('pymongo_test') def executed_on_which_server(self, client, fn, *args, **kwargs): """Execute fn(*args, **kwargs) and return the Server instance used.""" client.has_read_from.clear() fn(*args, **kwargs) self.assertEqual(1, len(client.has_read_from)) return one(client.has_read_from) def assertExecutedOn(self, server_type, client, fn, *args, **kwargs): server = self.executed_on_which_server(client, fn, *args, **kwargs) self.assertEqual(SERVER_TYPE._fields[server_type], SERVER_TYPE._fields[server.description.server_type]) def _test_fn(self, server_type, fn): for _ in range(10): if server_type == 'any': used = set() for _ in range(1000): server = self.executed_on_which_server(self.c, fn) used.add(server.description.address) if len(used) == len(self.c.secondaries) + 1: # Success break unused = self.c.secondaries.union( set([self.c.primary]) ).difference(used) if unused: self.fail( "Some members not used for NEAREST: %s" % ( unused)) else: self.assertExecutedOn(server_type, self.c, fn) def _test_primary_helper(self, func): # Helpers that ignore read preference. self._test_fn(SERVER_TYPE.RSPrimary, func) def _test_coll_helper(self, secondary_ok, coll, meth, *args, **kwargs): for mode, server_type in _PREF_MAP: new_coll = coll.with_options(read_preference=mode()) func = lambda: getattr(new_coll, meth)(*args, **kwargs) if secondary_ok: self._test_fn(server_type, func) else: self._test_fn(SERVER_TYPE.RSPrimary, func) def test_command(self): # Test that the generic command helper obeys the read preference # passed to it. for mode, server_type in _PREF_MAP: func = lambda: self.c.pymongo_test.command('dbStats', read_preference=mode()) self._test_fn(server_type, func) def test_create_collection(self): # Collections should be created on primary, obviously self._test_primary_helper( lambda: self.c.pymongo_test.create_collection( 'some_collection%s' % random.randint(0, MAXSIZE))) def test_drop_collection(self): self._test_primary_helper( lambda: self.c.pymongo_test.drop_collection('some_collection')) self._test_primary_helper( lambda: self.c.pymongo_test.some_collection.drop()) def test_group(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") self._test_coll_helper(True, self.c.pymongo_test.test, 'group', {'a': 1}, {}, {}, 'function() { }') def test_map_reduce(self): self._test_coll_helper(False, self.c.pymongo_test.test, 'map_reduce', 'function() { }', 'function() { }', {'inline': 1}) def test_inline_map_reduce(self): self._test_coll_helper(True, self.c.pymongo_test.test, 'inline_map_reduce', 'function() { }', 'function() { }') def test_count(self): self._test_coll_helper(True, self.c.pymongo_test.test, 'count') def test_distinct(self): self._test_coll_helper(True, self.c.pymongo_test.test, 'distinct', 'a') def test_aggregate(self): self._test_coll_helper(True, self.c.pymongo_test.test, 'aggregate', [{'$project': {'_id': 1}}]) class TestMovingAverage(unittest.TestCase): def test_moving_average(self): avg = MovingAverage() self.assertIsNone(avg.get()) avg.add_sample(10) self.assertAlmostEqual(10, avg.get()) avg.add_sample(20) self.assertAlmostEqual(12, avg.get()) avg.add_sample(30) self.assertAlmostEqual(15.6, avg.get()) class TestMongosAndReadPreference(unittest.TestCase): def test_read_preference_document(self): pref = Primary() self.assertEqual( pref.document, {'mode': 'primary'}) pref = PrimaryPreferred() self.assertEqual( pref.document, {'mode': 'primaryPreferred'}) pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}]) self.assertEqual( pref.document, {'mode': 'primaryPreferred', 'tags': [{'dc': 'sf'}]}) pref = PrimaryPreferred( tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual( pref.document, {'mode': 'primaryPreferred', 'tags': [{'dc': 'sf'}], 'maxStalenessSeconds': 30}) pref = Secondary() self.assertEqual( pref.document, {'mode': 'secondary'}) pref = Secondary(tag_sets=[{'dc': 'sf'}]) self.assertEqual( pref.document, {'mode': 'secondary', 'tags': [{'dc': 'sf'}]}) pref = Secondary( tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual( pref.document, {'mode': 'secondary', 'tags': [{'dc': 'sf'}], 'maxStalenessSeconds': 30}) pref = SecondaryPreferred() self.assertEqual( pref.document, {'mode': 'secondaryPreferred'}) pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}]) self.assertEqual( pref.document, {'mode': 'secondaryPreferred', 'tags': [{'dc': 'sf'}]}) pref = SecondaryPreferred( tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual( pref.document, {'mode': 'secondaryPreferred', 'tags': [{'dc': 'sf'}], 'maxStalenessSeconds': 30}) pref = Nearest() self.assertEqual( pref.document, {'mode': 'nearest'}) pref = Nearest(tag_sets=[{'dc': 'sf'}]) self.assertEqual( pref.document, {'mode': 'nearest', 'tags': [{'dc': 'sf'}]}) pref = Nearest( tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual( pref.document, {'mode': 'nearest', 'tags': [{'dc': 'sf'}], 'maxStalenessSeconds': 30}) with self.assertRaises(TypeError): Nearest(max_staleness=1.5) # Float is prohibited. with self.assertRaises(ValueError): Nearest(max_staleness=0) with self.assertRaises(ValueError): Nearest(max_staleness=-2) def test_maybe_add_read_preference(self): # Primary doesn't add $readPreference out = _maybe_add_read_preference({}, Primary()) self.assertEqual(out, {}) pref = PrimaryPreferred() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = PrimaryPreferred(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Secondary() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Secondary(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) # SecondaryPreferred without tag_sets or max_staleness doesn't add # $readPreference pref = SecondaryPreferred() out = _maybe_add_read_preference({}, pref) self.assertEqual(out, {}) pref = SecondaryPreferred(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = SecondaryPreferred(max_staleness=120) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Nearest() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Nearest(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) criteria = SON([("$query", {}), ("$orderby", SON([("_id", 1)]))]) pref = Nearest() out = _maybe_add_read_preference(criteria, pref) self.assertEqual( out, SON([("$query", {}), ("$orderby", SON([("_id", 1)])), ("$readPreference", pref.document)])) pref = Nearest(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference(criteria, pref) self.assertEqual( out, SON([("$query", {}), ("$orderby", SON([("_id", 1)])), ("$readPreference", pref.document)])) @client_context.require_mongos def test_mongos(self): shard = client_context.client.config.shards.find_one()['host'] num_members = shard.count(',') + 1 if num_members == 1: raise SkipTest("Need a replica set shard to test.") coll = client_context.client.pymongo_test.get_collection( "test", write_concern=WriteConcern(w=num_members)) coll.drop() res = coll.insert_many([{} for _ in range(5)]) first_id = res.inserted_ids[0] last_id = res.inserted_ids[-1] # Note - this isn't a perfect test since there's no way to # tell what shard member a query ran on. for pref in (Primary(), PrimaryPreferred(), Secondary(), SecondaryPreferred(), Nearest()): qcoll = coll.with_options(read_preference=pref) results = list(qcoll.find().sort([("_id", 1)])) self.assertEqual(first_id, results[0]["_id"]) self.assertEqual(last_id, results[-1]["_id"]) results = list(qcoll.find().sort([("_id", -1)])) self.assertEqual(first_id, results[-1]["_id"]) self.assertEqual(last_id, results[0]["_id"]) @client_context.require_mongos @client_context.require_version_min(3, 3, 12) def test_mongos_max_staleness(self): # Sanity check that we're sending maxStalenessSeconds coll = client_context.client.pymongo_test.get_collection( "test", read_preference=SecondaryPreferred(max_staleness=120)) # No error coll.find_one() coll = client_context.client.pymongo_test.get_collection( "test", read_preference=SecondaryPreferred(max_staleness=10)) try: coll.find_one() except OperationFailure as exc: self.assertEqual(160, exc.code) else: self.fail("mongos accepted invalid staleness") coll = single_client( readPreference='secondaryPreferred', maxStalenessSeconds=120).pymongo_test.test # No error coll.find_one() coll = single_client( readPreference='secondaryPreferred', maxStalenessSeconds=10).pymongo_test.test try: coll.find_one() except OperationFailure as exc: self.assertEqual(160, exc.code) else: self.fail("mongos accepted invalid staleness") if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_server_selection_rtt.py0000644000076600000240000000402013245617773022302 0ustar shanestaff00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the topology module.""" import json import os import sys sys.path[0:0] = [""] from test import unittest from pymongo.read_preferences import MovingAverage # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'server_selection/rtt') class TestAllScenarios(unittest.TestCase): pass def create_test(scenario_def): def run_scenario(self): moving_average = MovingAverage() if scenario_def['avg_rtt_ms'] != "NULL": moving_average.add_sample(scenario_def['avg_rtt_ms']) if scenario_def['new_rtt_ms'] != "NULL": moving_average.add_sample(scenario_def['new_rtt_ms']) self.assertAlmostEqual(moving_average.get(), scenario_def['new_avg_rtt']) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): dirname = os.path.split(dirpath)[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json.load(scenario_stream) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s_%s' % ( dirname, os.path.splitext(filename)[0]) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_monitor.py0000644000076600000240000000274513245621354017527 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the monitor module.""" import gc import sys from functools import partial sys.path[0:0] = [""] from pymongo.periodic_executor import _EXECUTORS from test import client_context, unittest, IntegrationTest from test.utils import single_client, one, connected, wait_until def unregistered(ref): gc.collect() return ref not in _EXECUTORS class TestMonitor(IntegrationTest): def test_atexit_hook(self): client = single_client(client_context.host, client_context.port) executor = one(client._topology._servers.values())._monitor._executor connected(client) # The executor stores a weakref to itself in _EXECUTORS. ref = one([r for r in _EXECUTORS.copy() if r() is executor]) del executor del client wait_until(partial(unregistered, ref), 'unregister executor', timeout=5) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_max_staleness.py0000644000076600000240000001320313245617773020707 0ustar shanestaff00000000000000# Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test maxStalenessSeconds support.""" import os import sys import time import warnings sys.path[0:0] = [""] from pymongo import MongoClient from pymongo.errors import ConfigurationError from pymongo.server_selectors import writable_server_selector from test import client_context, unittest from test.utils import rs_or_single_client from test.utils_selection_tests import create_selection_tests # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'max_staleness') class TestAllScenarios(create_selection_tests(_TEST_PATH)): pass class TestMaxStaleness(unittest.TestCase): def test_max_staleness(self): client = MongoClient() self.assertEqual(-1, client.read_preference.max_staleness) client = MongoClient("mongodb://a/?readPreference=secondary") self.assertEqual(-1, client.read_preference.max_staleness) # These tests are specified in max-staleness-tests.rst. with self.assertRaises(ConfigurationError): # Default read pref "primary" can't be used with max staleness. MongoClient("mongodb://a/?maxStalenessSeconds=120") with self.assertRaises(ConfigurationError): # Read pref "primary" can't be used with max staleness. MongoClient("mongodb://a/?readPreference=primary&" "maxStalenessSeconds=120") client = MongoClient("mongodb://host/?maxStalenessSeconds=-1") self.assertEqual(-1, client.read_preference.max_staleness) client = MongoClient("mongodb://host/?readPreference=primary&" "maxStalenessSeconds=-1") self.assertEqual(-1, client.read_preference.max_staleness) client = MongoClient("mongodb://host/?readPreference=secondary&" "maxStalenessSeconds=120") self.assertEqual(120, client.read_preference.max_staleness) client = MongoClient("mongodb://a/?readPreference=secondary&" "maxStalenessSeconds=1") self.assertEqual(1, client.read_preference.max_staleness) client = MongoClient("mongodb://a/?readPreference=secondary&" "maxStalenessSeconds=-1") self.assertEqual(-1, client.read_preference.max_staleness) client = MongoClient(maxStalenessSeconds=-1, readPreference="nearest") self.assertEqual(-1, client.read_preference.max_staleness) with self.assertRaises(TypeError): # Prohibit None. MongoClient(maxStalenessSeconds=None, readPreference="nearest") def test_max_staleness_float(self): with self.assertRaises(TypeError) as ctx: rs_or_single_client(maxStalenessSeconds=1.5, readPreference="nearest") self.assertIn("must be an integer", str(ctx.exception)) with warnings.catch_warnings(record=True) as ctx: warnings.simplefilter("always") client = MongoClient("mongodb://host/?maxStalenessSeconds=1.5" "&readPreference=nearest") # Option was ignored. self.assertEqual(-1, client.read_preference.max_staleness) self.assertIn("must be an integer", str(ctx[0])) def test_max_staleness_zero(self): # Zero is too small. with self.assertRaises(ValueError) as ctx: rs_or_single_client(maxStalenessSeconds=0, readPreference="nearest") self.assertIn("must be a positive integer", str(ctx.exception)) with warnings.catch_warnings(record=True) as ctx: warnings.simplefilter("always") client = MongoClient("mongodb://host/?maxStalenessSeconds=0" "&readPreference=nearest") # Option was ignored. self.assertEqual(-1, client.read_preference.max_staleness) self.assertIn("must be a positive integer", str(ctx[0])) @client_context.require_version_min(3, 3, 6) # SERVER-8858 @client_context.require_replica_set def test_last_write_date(self): # From max-staleness-tests.rst, "Parse lastWriteDate". client = rs_or_single_client(heartbeatFrequencyMS=500) client.pymongo_test.test.insert_one({}) time.sleep(1) server = client._topology.select_server(writable_server_selector) last_write = server.description.last_write_date self.assertTrue(last_write) client.pymongo_test.test.insert_one({}) time.sleep(1) server = client._topology.select_server(writable_server_selector) self.assertGreater(server.description.last_write_date, last_write) self.assertLess(server.description.last_write_date, last_write + 10) @client_context.require_version_max(3, 3) def test_last_write_date_absent(self): # From max-staleness-tests.rst, "Absent lastWriteDate". client = rs_or_single_client() sd = client._topology.select_server(writable_server_selector) self.assertIsNone(sd.description.last_write_date) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_bson.py0000644000076600000240000012344213245621354016777 0ustar shanestaff00000000000000# -*- coding: utf-8 -*- # # Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the bson module.""" import collections import datetime import re import sys import uuid sys.path[0:0] = [""] import bson from bson import (BSON, decode_all, decode_file_iter, decode_iter, EPOCH_AWARE, is_valid, Regex) from bson.binary import Binary, UUIDLegacy from bson.code import Code from bson.codec_options import CodecOptions from bson.int64 import Int64 from bson.objectid import ObjectId from bson.dbref import DBRef from bson.py3compat import abc, iteritems, PY3, StringIO, text_type from bson.son import SON from bson.timestamp import Timestamp from bson.tz_util import FixedOffset from bson.errors import (InvalidBSON, InvalidDocument, InvalidStringData) from bson.max_key import MaxKey from bson.min_key import MinKey from bson.tz_util import (FixedOffset, utc) from test import qcheck, SkipTest, unittest if PY3: long = int class NotADict(abc.MutableMapping): """Non-dict type that implements the mapping protocol.""" def __init__(self, initial=None): if not initial: self._dict = {} else: self._dict = initial def __iter__(self): return iter(self._dict) def __getitem__(self, item): return self._dict[item] def __delitem__(self, item): del self._dict[item] def __setitem__(self, item, value): self._dict[item] = value def __len__(self): return len(self._dict) def __eq__(self, other): if isinstance(other, abc.Mapping): return all(self.get(k) == other.get(k) for k in self) return NotImplemented def __repr__(self): return "NotADict(%s)" % repr(self._dict) class DSTAwareTimezone(datetime.tzinfo): def __init__(self, offset, name, dst_start_month, dst_end_month): self.__offset = offset self.__dst_start_month = dst_start_month self.__dst_end_month = dst_end_month self.__name = name def _is_dst(self, dt): return self.__dst_start_month <= dt.month <= self.__dst_end_month def utcoffset(self, dt): return datetime.timedelta(minutes=self.__offset) + self.dst(dt) def dst(self, dt): if self._is_dst(dt): return datetime.timedelta(hours=1) return datetime.timedelta(0) def tzname(self, dt): return self.__name class TestBSON(unittest.TestCase): def assertInvalid(self, data): self.assertRaises(InvalidBSON, bson.BSON(data).decode) def check_encode_then_decode(self, doc_class=dict): # Work around http://bugs.jython.org/issue1728 if sys.platform.startswith('java'): doc_class = SON def helper(doc): self.assertEqual(doc, (BSON.encode(doc_class(doc))).decode()) helper({}) helper({"test": u"hello"}) self.assertTrue(isinstance(BSON.encode({"hello": "world"}) .decode()["hello"], text_type)) helper({"mike": -10120}) helper({"long": Int64(10)}) helper({"really big long": 2147483648}) helper({u"hello": 0.0013109}) helper({"something": True}) helper({"false": False}) helper({"an array": [1, True, 3.8, u"world"]}) helper({"an object": doc_class({"test": u"something"})}) helper({"a binary": Binary(b"test", 100)}) helper({"a binary": Binary(b"test", 128)}) helper({"a binary": Binary(b"test", 254)}) helper({"another binary": Binary(b"test", 2)}) helper(SON([(u'test dst', datetime.datetime(1993, 4, 4, 2))])) helper(SON([(u'test negative dst', datetime.datetime(1, 1, 1, 1, 1, 1))])) helper({"big float": float(10000000000)}) helper({"ref": DBRef("coll", 5)}) helper({"ref": DBRef("coll", 5, foo="bar", bar=4)}) helper({"ref": DBRef("coll", 5, "foo")}) helper({"ref": DBRef("coll", 5, "foo", foo="bar")}) helper({"ref": Timestamp(1, 2)}) helper({"foo": MinKey()}) helper({"foo": MaxKey()}) helper({"$field": Code("function(){ return true; }")}) helper({"$field": Code("return function(){ return x; }", scope={'x': False})}) def encode_then_decode(doc): return doc_class(doc) == BSON.encode(doc).decode( CodecOptions(document_class=doc_class)) qcheck.check_unittest(self, encode_then_decode, qcheck.gen_mongo_dict(3)) def test_encode_then_decode(self): self.check_encode_then_decode() def test_encode_then_decode_any_mapping(self): self.check_encode_then_decode(doc_class=NotADict) def test_encoding_defaultdict(self): dct = collections.defaultdict(dict, [('foo', 'bar')]) BSON.encode(dct) self.assertEqual(dct, collections.defaultdict(dict, [('foo', 'bar')])) def test_basic_validation(self): self.assertRaises(TypeError, is_valid, 100) self.assertRaises(TypeError, is_valid, u"test") self.assertRaises(TypeError, is_valid, 10.4) self.assertInvalid(b"test") # the simplest valid BSON document self.assertTrue(is_valid(b"\x05\x00\x00\x00\x00")) self.assertTrue(is_valid(BSON(b"\x05\x00\x00\x00\x00"))) # failure cases self.assertInvalid(b"\x04\x00\x00\x00\x00") self.assertInvalid(b"\x05\x00\x00\x00\x01") self.assertInvalid(b"\x05\x00\x00\x00") self.assertInvalid(b"\x05\x00\x00\x00\x00\x00") self.assertInvalid(b"\x07\x00\x00\x00\x02a\x00\x78\x56\x34\x12") self.assertInvalid(b"\x09\x00\x00\x00\x10a\x00\x05\x00") self.assertInvalid(b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00") self.assertInvalid(b"\x13\x00\x00\x00\x02foo\x00" b"\x04\x00\x00\x00bar\x00\x00") self.assertInvalid(b"\x18\x00\x00\x00\x03foo\x00\x0f\x00\x00" b"\x00\x10bar\x00\xff\xff\xff\x7f\x00\x00") self.assertInvalid(b"\x15\x00\x00\x00\x03foo\x00\x0c" b"\x00\x00\x00\x08bar\x00\x01\x00\x00") self.assertInvalid(b"\x1c\x00\x00\x00\x03foo\x00" b"\x12\x00\x00\x00\x02bar\x00" b"\x05\x00\x00\x00baz\x00\x00\x00") self.assertInvalid(b"\x10\x00\x00\x00\x02a\x00" b"\x04\x00\x00\x00abc\xff\x00") def test_bad_string_lengths(self): self.assertInvalid( b"\x0c\x00\x00\x00\x02\x00" b"\x00\x00\x00\x00\x00\x00") self.assertInvalid( b"\x12\x00\x00\x00\x02\x00" b"\xff\xff\xff\xfffoobar\x00\x00") self.assertInvalid( b"\x0c\x00\x00\x00\x0e\x00" b"\x00\x00\x00\x00\x00\x00") self.assertInvalid( b"\x12\x00\x00\x00\x0e\x00" b"\xff\xff\xff\xfffoobar\x00\x00") self.assertInvalid( b"\x18\x00\x00\x00\x0c\x00" b"\x00\x00\x00\x00\x00RY\xb5j" b"\xfa[\xd8A\xd6X]\x99\x00") self.assertInvalid( b"\x1e\x00\x00\x00\x0c\x00" b"\xff\xff\xff\xfffoobar\x00" b"RY\xb5j\xfa[\xd8A\xd6X]\x99\x00") self.assertInvalid( b"\x0c\x00\x00\x00\r\x00" b"\x00\x00\x00\x00\x00\x00") self.assertInvalid( b"\x0c\x00\x00\x00\r\x00" b"\xff\xff\xff\xff\x00\x00") self.assertInvalid( b"\x1c\x00\x00\x00\x0f\x00" b"\x15\x00\x00\x00\x00\x00" b"\x00\x00\x00\x0c\x00\x00" b"\x00\x02\x00\x01\x00\x00" b"\x00\x00\x00\x00") self.assertInvalid( b"\x1c\x00\x00\x00\x0f\x00" b"\x15\x00\x00\x00\xff\xff" b"\xff\xff\x00\x0c\x00\x00" b"\x00\x02\x00\x01\x00\x00" b"\x00\x00\x00\x00") self.assertInvalid( b"\x1c\x00\x00\x00\x0f\x00" b"\x15\x00\x00\x00\x01\x00" b"\x00\x00\x00\x0c\x00\x00" b"\x00\x02\x00\x00\x00\x00" b"\x00\x00\x00\x00") self.assertInvalid( b"\x1c\x00\x00\x00\x0f\x00" b"\x15\x00\x00\x00\x01\x00" b"\x00\x00\x00\x0c\x00\x00" b"\x00\x02\x00\xff\xff\xff" b"\xff\x00\x00\x00") def test_random_data_is_not_bson(self): qcheck.check_unittest(self, qcheck.isnt(is_valid), qcheck.gen_string(qcheck.gen_range(0, 40))) def test_basic_decode(self): self.assertEqual({"test": u"hello world"}, BSON(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74\x00\x0C" b"\x00\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F" b"\x72\x6C\x64\x00\x00").decode()) self.assertEqual([{"test": u"hello world"}, {}], decode_all(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\x00")) self.assertEqual([{"test": u"hello world"}, {}], list(decode_iter( b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\x00"))) self.assertEqual([{"test": u"hello world"}, {}], list(decode_file_iter(StringIO( b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\x00")))) def test_invalid_decodes(self): # Invalid object size (not enough bytes in document for even # an object size of first object. # NOTE: decode_all and decode_iter don't care, not sure if they should? self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(b"\x1B"))) # An object size that's too small to even include the object size, # but is correctly encoded, along with a correct EOO (and no data). data = b"\x01\x00\x00\x00\x00" self.assertRaises(InvalidBSON, decode_all, data) self.assertRaises(InvalidBSON, list, decode_iter(data)) self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data))) # One object, but with object size listed smaller than it is in the # data. data = (b"\x1A\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\x00") self.assertRaises(InvalidBSON, decode_all, data) self.assertRaises(InvalidBSON, list, decode_iter(data)) self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data))) # One object, missing the EOO at the end. data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00") self.assertRaises(InvalidBSON, decode_all, data) self.assertRaises(InvalidBSON, list, decode_iter(data)) self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data))) # One object, sized correctly, with a spot for an EOO, but the EOO # isn't 0x00. data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\xFF") self.assertRaises(InvalidBSON, decode_all, data) self.assertRaises(InvalidBSON, list, decode_iter(data)) self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data))) def test_data_timestamp(self): self.assertEqual({"test": Timestamp(4, 20)}, BSON(b"\x13\x00\x00\x00\x11\x74\x65\x73\x74\x00\x14" b"\x00\x00\x00\x04\x00\x00\x00\x00").decode()) def test_basic_encode(self): self.assertRaises(TypeError, BSON.encode, 100) self.assertRaises(TypeError, BSON.encode, "hello") self.assertRaises(TypeError, BSON.encode, None) self.assertRaises(TypeError, BSON.encode, []) self.assertEqual(BSON.encode({}), BSON(b"\x05\x00\x00\x00\x00")) self.assertEqual(BSON.encode({"test": u"hello world"}), b"\x1B\x00\x00\x00\x02\x74\x65\x73\x74\x00\x0C\x00" b"\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F\x72\x6C" b"\x64\x00\x00") self.assertEqual(BSON.encode({u"mike": 100}), b"\x0F\x00\x00\x00\x10\x6D\x69\x6B\x65\x00\x64\x00" b"\x00\x00\x00") self.assertEqual(BSON.encode({"hello": 1.5}), b"\x14\x00\x00\x00\x01\x68\x65\x6C\x6C\x6F\x00\x00" b"\x00\x00\x00\x00\x00\xF8\x3F\x00") self.assertEqual(BSON.encode({"true": True}), b"\x0C\x00\x00\x00\x08\x74\x72\x75\x65\x00\x01\x00") self.assertEqual(BSON.encode({"false": False}), b"\x0D\x00\x00\x00\x08\x66\x61\x6C\x73\x65\x00\x00" b"\x00") self.assertEqual(BSON.encode({"empty": []}), b"\x11\x00\x00\x00\x04\x65\x6D\x70\x74\x79\x00\x05" b"\x00\x00\x00\x00\x00") self.assertEqual(BSON.encode({"none": {}}), b"\x10\x00\x00\x00\x03\x6E\x6F\x6E\x65\x00\x05\x00" b"\x00\x00\x00\x00") self.assertEqual(BSON.encode({"test": Binary(b"test", 0)}), b"\x14\x00\x00\x00\x05\x74\x65\x73\x74\x00\x04\x00" b"\x00\x00\x00\x74\x65\x73\x74\x00") self.assertEqual(BSON.encode({"test": Binary(b"test", 2)}), b"\x18\x00\x00\x00\x05\x74\x65\x73\x74\x00\x08\x00" b"\x00\x00\x02\x04\x00\x00\x00\x74\x65\x73\x74\x00") self.assertEqual(BSON.encode({"test": Binary(b"test", 128)}), b"\x14\x00\x00\x00\x05\x74\x65\x73\x74\x00\x04\x00" b"\x00\x00\x80\x74\x65\x73\x74\x00") self.assertEqual(BSON.encode({"test": None}), b"\x0B\x00\x00\x00\x0A\x74\x65\x73\x74\x00\x00") self.assertEqual(BSON.encode({"date": datetime.datetime(2007, 1, 8, 0, 30, 11)}), b"\x13\x00\x00\x00\x09\x64\x61\x74\x65\x00\x38\xBE" b"\x1C\xFF\x0F\x01\x00\x00\x00") self.assertEqual(BSON.encode({"regex": re.compile(b"a*b", re.IGNORECASE)}), b"\x12\x00\x00\x00\x0B\x72\x65\x67\x65\x78\x00\x61" b"\x2A\x62\x00\x69\x00\x00") self.assertEqual(BSON.encode({"$where": Code("test")}), b"\x16\x00\x00\x00\r$where\x00\x05\x00\x00\x00test" b"\x00\x00") self.assertEqual(BSON.encode({"$field": Code("function(){ return true;}", scope=None)}), b"+\x00\x00\x00\r$field\x00\x1a\x00\x00\x00" b"function(){ return true;}\x00\x00") self.assertEqual(BSON.encode({"$field": Code("return function(){ return x; }", scope={'x': False})}), b"=\x00\x00\x00\x0f$field\x000\x00\x00\x00\x1f\x00" b"\x00\x00return function(){ return x; }\x00\t\x00" b"\x00\x00\x08x\x00\x00\x00\x00") unicode_empty_scope = Code(u"function(){ return 'héllo';}", {}) self.assertEqual(BSON.encode({'$field': unicode_empty_scope}), b"8\x00\x00\x00\x0f$field\x00+\x00\x00\x00\x1e\x00" b"\x00\x00function(){ return 'h\xc3\xa9llo';}\x00\x05" b"\x00\x00\x00\x00\x00") a = ObjectId(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B") self.assertEqual(BSON.encode({"oid": a}), b"\x16\x00\x00\x00\x07\x6F\x69\x64\x00\x00\x01\x02" b"\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x00") self.assertEqual(BSON.encode({"ref": DBRef("coll", a)}), b"\x2F\x00\x00\x00\x03ref\x00\x25\x00\x00\x00\x02" b"$ref\x00\x05\x00\x00\x00coll\x00\x07$id\x00\x00" b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x00" b"\x00") def test_unknown_type(self): # Repr value differs with major python version part = "type %r for fieldname 'foo'" % (b'\x14',) docs = [ b'\x0e\x00\x00\x00\x14foo\x00\x01\x00\x00\x00\x00', (b'\x16\x00\x00\x00\x04foo\x00\x0c\x00\x00\x00\x140' b'\x00\x01\x00\x00\x00\x00\x00'), (b' \x00\x00\x00\x04bar\x00\x16\x00\x00\x00\x030\x00\x0e\x00\x00' b'\x00\x14foo\x00\x01\x00\x00\x00\x00\x00\x00')] for bs in docs: try: bson.BSON(bs).decode() except Exception as exc: self.assertTrue(isinstance(exc, InvalidBSON)) self.assertTrue(part in str(exc)) else: self.fail("Failed to raise an exception.") def test_dbpointer(self): # *Note* - DBPointer and DBRef are *not* the same thing. DBPointer # is a deprecated BSON type. DBRef is a convention that does not # exist in the BSON spec, meant to replace DBPointer. PyMongo does # not support creation of the DBPointer type, but will decode # DBPointer to DBRef. bs = (b"\x18\x00\x00\x00\x0c\x00\x01\x00\x00" b"\x00\x00RY\xb5j\xfa[\xd8A\xd6X]\x99\x00") self.assertEqual({'': DBRef('', ObjectId('5259b56afa5bd841d6585d99'))}, bson.BSON(bs).decode()) def test_bad_dbref(self): ref_only = {'ref': {'$ref': 'collection'}} id_only = {'ref': {'$id': ObjectId()}} self.assertEqual(DBRef('collection', id=None), BSON.encode(ref_only).decode()['ref']) self.assertEqual(id_only, BSON.encode(id_only).decode()) def test_bytes_as_keys(self): doc = {b"foo": 'bar'} # Since `bytes` are stored as Binary you can't use them # as keys in python 3.x. Using binary data as a key makes # no sense in BSON anyway and little sense in python. if PY3: self.assertRaises(InvalidDocument, BSON.encode, doc) else: self.assertTrue(BSON.encode(doc)) def test_datetime_encode_decode(self): # Negative timestamps dt1 = datetime.datetime(1, 1, 1, 1, 1, 1, 111000) dt2 = BSON.encode({"date": dt1}).decode()["date"] self.assertEqual(dt1, dt2) dt1 = datetime.datetime(1959, 6, 25, 12, 16, 59, 999000) dt2 = BSON.encode({"date": dt1}).decode()["date"] self.assertEqual(dt1, dt2) # Positive timestamps dt1 = datetime.datetime(9999, 12, 31, 23, 59, 59, 999000) dt2 = BSON.encode({"date": dt1}).decode()["date"] self.assertEqual(dt1, dt2) dt1 = datetime.datetime(2011, 6, 14, 10, 47, 53, 444000) dt2 = BSON.encode({"date": dt1}).decode()["date"] self.assertEqual(dt1, dt2) def test_aware_datetime(self): aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc) self.assertEqual(datetime.datetime(1993, 4, 3, 16, 45, tzinfo=utc), as_utc) after = BSON.encode({"date": aware}).decode( CodecOptions(tz_aware=True))["date"] self.assertEqual(utc, after.tzinfo) self.assertEqual(as_utc, after) def test_local_datetime(self): # Timezone -60 minutes of UTC, with DST between April and July. tz = DSTAwareTimezone(60, "sixty-minutes", 4, 7) # It's not DST. local = datetime.datetime(year=2025, month=12, hour=2, day=1, tzinfo=tz) options = CodecOptions(tz_aware=True, tzinfo=tz) # Encode with this timezone, then decode to UTC. encoded = BSON.encode({'date': local}, codec_options=options) self.assertEqual(local.replace(hour=1, tzinfo=None), encoded.decode()['date']) # It's DST. local = datetime.datetime(year=2025, month=4, hour=1, day=1, tzinfo=tz) encoded = BSON.encode({'date': local}, codec_options=options) self.assertEqual(local.replace(month=3, day=31, hour=23, tzinfo=None), encoded.decode()['date']) # Encode UTC, then decode in a different timezone. encoded = BSON.encode({'date': local.replace(tzinfo=utc)}) decoded = encoded.decode(options)['date'] self.assertEqual(local.replace(hour=3), decoded) self.assertEqual(tz, decoded.tzinfo) # Test round-tripping. self.assertEqual( local, (BSON .encode({'date': local}, codec_options=options) .decode(options)['date'])) # Test around the Unix Epoch. epochs = ( EPOCH_AWARE, EPOCH_AWARE.astimezone(FixedOffset(120, 'one twenty')), EPOCH_AWARE.astimezone(FixedOffset(-120, 'minus one twenty')) ) utc_co = CodecOptions(tz_aware=True) for epoch in epochs: doc = {'epoch': epoch} # We always retrieve datetimes in UTC unless told to do otherwise. self.assertEqual( EPOCH_AWARE, BSON.encode(doc).decode(codec_options=utc_co)['epoch']) # Round-trip the epoch. local_co = CodecOptions(tz_aware=True, tzinfo=epoch.tzinfo) self.assertEqual( epoch, BSON.encode(doc).decode(codec_options=local_co)['epoch']) def test_naive_decode(self): aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) naive_utc = (aware - aware.utcoffset()).replace(tzinfo=None) self.assertEqual(datetime.datetime(1993, 4, 3, 16, 45), naive_utc) after = BSON.encode({"date": aware}).decode()["date"] self.assertEqual(None, after.tzinfo) self.assertEqual(naive_utc, after) def test_dst(self): d = {"x": datetime.datetime(1993, 4, 4, 2)} self.assertEqual(d, BSON.encode(d).decode()) def test_bad_encode(self): if not PY3: # Python3 treats this as a unicode string which won't raise # an exception. If we passed the string as bytes instead we # still wouldn't get an error since we store bytes as BSON # binary subtype 0. self.assertRaises(InvalidStringData, BSON.encode, {"lalala": '\xf4\xe0\xf0\xe1\xc0 Color Touch'}) # Work around what seems like a regression in python 3.5.0. # See http://bugs.python.org/issue25222 if sys.version_info[:2] < (3, 5): evil_list = {'a': []} evil_list['a'].append(evil_list) evil_dict = {} evil_dict['a'] = evil_dict for evil_data in [evil_dict, evil_list]: self.assertRaises(Exception, BSON.encode, evil_data) def test_overflow(self): self.assertTrue(BSON.encode({"x": long(9223372036854775807)})) self.assertRaises(OverflowError, BSON.encode, {"x": long(9223372036854775808)}) self.assertTrue(BSON.encode({"x": long(-9223372036854775808)})) self.assertRaises(OverflowError, BSON.encode, {"x": long(-9223372036854775809)}) def test_small_long_encode_decode(self): encoded1 = BSON.encode({'x': 256}) decoded1 = BSON.decode(encoded1)['x'] self.assertEqual(256, decoded1) self.assertEqual(type(256), type(decoded1)) encoded2 = BSON.encode({'x': Int64(256)}) decoded2 = BSON.decode(encoded2)['x'] expected = Int64(256) self.assertEqual(expected, decoded2) self.assertEqual(type(expected), type(decoded2)) self.assertNotEqual(type(decoded1), type(decoded2)) def test_tuple(self): self.assertEqual({"tuple": [1, 2]}, BSON.encode({"tuple": (1, 2)}).decode()) def test_uuid(self): id = uuid.uuid4() transformed_id = (BSON.encode({"id": id})).decode()["id"] self.assertTrue(isinstance(transformed_id, uuid.UUID)) self.assertEqual(id, transformed_id) self.assertNotEqual(uuid.uuid4(), transformed_id) def test_uuid_legacy(self): id = uuid.uuid4() legacy = UUIDLegacy(id) self.assertEqual(3, legacy.subtype) transformed = (BSON.encode({"uuid": legacy})).decode()["uuid"] self.assertTrue(isinstance(transformed, uuid.UUID)) self.assertEqual(id, transformed) self.assertNotEqual(UUIDLegacy(uuid.uuid4()), UUIDLegacy(transformed)) # The C extension was segfaulting on unicode RegExs, so we have this test # that doesn't really test anything but the lack of a segfault. def test_unicode_regex(self): regex = re.compile(u'revisi\xf3n') BSON.encode({"regex": regex}).decode() def test_non_string_keys(self): self.assertRaises(InvalidDocument, BSON.encode, {8.9: "test"}) def test_utf8(self): w = {u"aéあ": u"aéあ"} self.assertEqual(w, BSON.encode(w).decode()) # b'a\xe9' == u"aé".encode("iso-8859-1") iso8859_bytes = b'a\xe9' y = {"hello": iso8859_bytes} if PY3: # Stored as BSON binary subtype 0. out = BSON.encode(y).decode() self.assertTrue(isinstance(out['hello'], bytes)) self.assertEqual(out['hello'], iso8859_bytes) else: # Python 2. try: BSON.encode(y) except InvalidStringData as e: self.assertTrue(repr(iso8859_bytes) in str(e)) # The next two tests only make sense in python 2.x since # you can't use `bytes` type as document keys in python 3.x. x = {u"aéあ".encode("utf-8"): u"aéあ".encode("utf-8")} self.assertEqual(w, BSON.encode(x).decode()) z = {iso8859_bytes: "hello"} self.assertRaises(InvalidStringData, BSON.encode, z) def test_null_character(self): doc = {"a": "\x00"} self.assertEqual(doc, BSON.encode(doc).decode()) # This test doesn't make much sense in Python2 # since {'a': '\x00'} == {'a': u'\x00'}. # Decoding here actually returns {'a': '\x00'} doc = {"a": u"\x00"} self.assertEqual(doc, BSON.encode(doc).decode()) self.assertRaises(InvalidDocument, BSON.encode, {b"\x00": "a"}) self.assertRaises(InvalidDocument, BSON.encode, {u"\x00": "a"}) self.assertRaises(InvalidDocument, BSON.encode, {"a": re.compile(b"ab\x00c")}) self.assertRaises(InvalidDocument, BSON.encode, {"a": re.compile(u"ab\x00c")}) def test_move_id(self): self.assertEqual(b"\x19\x00\x00\x00\x02_id\x00\x02\x00\x00\x00a\x00" b"\x02a\x00\x02\x00\x00\x00a\x00\x00", BSON.encode(SON([("a", "a"), ("_id", "a")]))) self.assertEqual(b"\x2c\x00\x00\x00" b"\x02_id\x00\x02\x00\x00\x00b\x00" b"\x03b\x00" b"\x19\x00\x00\x00\x02a\x00\x02\x00\x00\x00a\x00" b"\x02_id\x00\x02\x00\x00\x00a\x00\x00\x00", BSON.encode(SON([("b", SON([("a", "a"), ("_id", "a")])), ("_id", "b")]))) def test_dates(self): doc = {"early": datetime.datetime(1686, 5, 5), "late": datetime.datetime(2086, 5, 5)} try: self.assertEqual(doc, BSON.encode(doc).decode()) except ValueError: # Ignore ValueError when no C ext, since it's probably # a problem w/ 32-bit Python - we work around this in the # C ext, though. if bson.has_c(): raise def test_custom_class(self): self.assertIsInstance(BSON.encode({}).decode(), dict) self.assertNotIsInstance(BSON.encode({}).decode(), SON) self.assertIsInstance( BSON.encode({}).decode(CodecOptions(document_class=SON)), SON) self.assertEqual( 1, BSON.encode({"x": 1}).decode( CodecOptions(document_class=SON))["x"]) x = BSON.encode({"x": [{"y": 1}]}) self.assertIsInstance( x.decode(CodecOptions(document_class=SON))["x"][0], SON) def test_subclasses(self): # make sure we can serialize subclasses of native Python types. class _myint(int): pass class _myfloat(float): pass class _myunicode(text_type): pass d = {'a': _myint(42), 'b': _myfloat(63.9), 'c': _myunicode('hello world') } d2 = BSON.encode(d).decode() for key, value in iteritems(d2): orig_value = d[key] orig_type = orig_value.__class__.__bases__[0] self.assertEqual(type(value), orig_type) self.assertEqual(value, orig_type(value)) def test_ordered_dict(self): try: from collections import OrderedDict except ImportError: raise SkipTest("No OrderedDict") d = OrderedDict([("one", 1), ("two", 2), ("three", 3), ("four", 4)]) self.assertEqual( d, BSON.encode(d).decode(CodecOptions(document_class=OrderedDict))) def test_bson_regex(self): # Invalid Python regex, though valid PCRE. bson_re1 = Regex(r'[\w-\.]') self.assertEqual(r'[\w-\.]', bson_re1.pattern) self.assertEqual(0, bson_re1.flags) doc1 = {'r': bson_re1} doc1_bson = ( b'\x11\x00\x00\x00' # document length b'\x0br\x00[\\w-\\.]\x00\x00' # r: regex b'\x00') # document terminator self.assertEqual(doc1_bson, BSON.encode(doc1)) self.assertEqual(doc1, BSON(doc1_bson).decode()) # Valid Python regex, with flags. re2 = re.compile(u'.*', re.I | re.M | re.S | re.U | re.X) bson_re2 = Regex(u'.*', re.I | re.M | re.S | re.U | re.X) doc2_with_re = {'r': re2} doc2_with_bson_re = {'r': bson_re2} doc2_bson = ( b"\x11\x00\x00\x00" # document length b"\x0br\x00.*\x00imsux\x00" # r: regex b"\x00") # document terminator self.assertEqual(doc2_bson, BSON.encode(doc2_with_re)) self.assertEqual(doc2_bson, BSON.encode(doc2_with_bson_re)) self.assertEqual(re2.pattern, BSON(doc2_bson).decode()['r'].pattern) self.assertEqual(re2.flags, BSON(doc2_bson).decode()['r'].flags) def test_regex_from_native(self): self.assertEqual('.*', Regex.from_native(re.compile('.*')).pattern) self.assertEqual(0, Regex.from_native(re.compile(b'')).flags) regex = re.compile(b'', re.I | re.L | re.M | re.S | re.X) self.assertEqual( re.I | re.L | re.M | re.S | re.X, Regex.from_native(regex).flags) unicode_regex = re.compile('', re.U) self.assertEqual(re.U, Regex.from_native(unicode_regex).flags) def test_regex_hash(self): self.assertRaises(TypeError, hash, Regex('hello')) def test_regex_comparison(self): re1 = Regex('a') re2 = Regex('b') self.assertNotEqual(re1, re2) re1 = Regex('a', re.I) re2 = Regex('a', re.M) self.assertNotEqual(re1, re2) re1 = Regex('a', re.I) re2 = Regex('a', re.I) self.assertEqual(re1, re2) def test_exception_wrapping(self): # No matter what exception is raised while trying to decode BSON, # the final exception always matches InvalidBSON. # {'s': '\xff'}, will throw attempting to decode utf-8. bad_doc = b'\x0f\x00\x00\x00\x02s\x00\x03\x00\x00\x00\xff\x00\x00\x00' with self.assertRaises(InvalidBSON) as context: decode_all(bad_doc) self.assertIn("codec can't decode byte 0xff", str(context.exception)) def test_minkey_maxkey_comparison(self): # MinKey's <, <=, >, >=, !=, and ==. self.assertTrue(MinKey() < None) self.assertTrue(MinKey() < 1) self.assertTrue(MinKey() <= 1) self.assertTrue(MinKey() <= MinKey()) self.assertFalse(MinKey() > None) self.assertFalse(MinKey() > 1) self.assertFalse(MinKey() >= 1) self.assertTrue(MinKey() >= MinKey()) self.assertTrue(MinKey() != 1) self.assertFalse(MinKey() == 1) self.assertTrue(MinKey() == MinKey()) # MinKey compared to MaxKey. self.assertTrue(MinKey() < MaxKey()) self.assertTrue(MinKey() <= MaxKey()) self.assertFalse(MinKey() > MaxKey()) self.assertFalse(MinKey() >= MaxKey()) self.assertTrue(MinKey() != MaxKey()) self.assertFalse(MinKey() == MaxKey()) # MaxKey's <, <=, >, >=, !=, and ==. self.assertFalse(MaxKey() < None) self.assertFalse(MaxKey() < 1) self.assertFalse(MaxKey() <= 1) self.assertTrue(MaxKey() <= MaxKey()) self.assertTrue(MaxKey() > None) self.assertTrue(MaxKey() > 1) self.assertTrue(MaxKey() >= 1) self.assertTrue(MaxKey() >= MaxKey()) self.assertTrue(MaxKey() != 1) self.assertFalse(MaxKey() == 1) self.assertTrue(MaxKey() == MaxKey()) # MaxKey compared to MinKey. self.assertFalse(MaxKey() < MinKey()) self.assertFalse(MaxKey() <= MinKey()) self.assertTrue(MaxKey() > MinKey()) self.assertTrue(MaxKey() >= MinKey()) self.assertTrue(MaxKey() != MinKey()) self.assertFalse(MaxKey() == MinKey()) def test_minkey_maxkey_hash(self): self.assertEqual(hash(MaxKey()), hash(MaxKey())) self.assertEqual(hash(MinKey()), hash(MinKey())) self.assertNotEqual(hash(MaxKey()), hash(MinKey())) def test_timestamp_comparison(self): # Timestamp is initialized with time, inc. Time is the more # significant comparand. self.assertTrue(Timestamp(1, 0) < Timestamp(2, 17)) self.assertTrue(Timestamp(2, 0) > Timestamp(1, 0)) self.assertTrue(Timestamp(1, 7) <= Timestamp(2, 0)) self.assertTrue(Timestamp(2, 0) >= Timestamp(1, 1)) self.assertTrue(Timestamp(2, 0) <= Timestamp(2, 0)) self.assertTrue(Timestamp(2, 0) >= Timestamp(2, 0)) self.assertFalse(Timestamp(1, 0) > Timestamp(2, 0)) # Comparison by inc. self.assertTrue(Timestamp(1, 0) < Timestamp(1, 1)) self.assertTrue(Timestamp(1, 1) > Timestamp(1, 0)) self.assertTrue(Timestamp(1, 0) <= Timestamp(1, 0)) self.assertTrue(Timestamp(1, 0) <= Timestamp(1, 1)) self.assertFalse(Timestamp(1, 0) >= Timestamp(1, 1)) self.assertTrue(Timestamp(1, 0) >= Timestamp(1, 0)) self.assertTrue(Timestamp(1, 1) >= Timestamp(1, 0)) self.assertFalse(Timestamp(1, 1) <= Timestamp(1, 0)) self.assertTrue(Timestamp(1, 0) <= Timestamp(1, 0)) self.assertFalse(Timestamp(1, 0) > Timestamp(1, 0)) def test_timestamp_highorder_bits(self): doc = {'a': Timestamp(0xFFFFFFFF, 0xFFFFFFFF)} doc_bson = (b'\x10\x00\x00\x00' b'\x11a\x00\xff\xff\xff\xff\xff\xff\xff\xff' b'\x00') self.assertEqual(doc_bson, BSON.encode(doc)) self.assertEqual(doc, BSON(doc_bson).decode()) def test_bad_id_keys(self): self.assertRaises(InvalidDocument, BSON.encode, {"_id": {"$bad": 123}}, True) self.assertRaises(InvalidDocument, BSON.encode, {"_id": {'$oid': "52d0b971b3ba219fdeb4170e"}}, True) BSON.encode({"_id": {'$oid': "52d0b971b3ba219fdeb4170e"}}) class TestCodecOptions(unittest.TestCase): def test_document_class(self): self.assertRaises(TypeError, CodecOptions, document_class=object) self.assertIs(SON, CodecOptions(document_class=SON).document_class) def test_tz_aware(self): self.assertRaises(TypeError, CodecOptions, tz_aware=1) self.assertFalse(CodecOptions().tz_aware) self.assertTrue(CodecOptions(tz_aware=True).tz_aware) def test_uuid_representation(self): self.assertRaises(ValueError, CodecOptions, uuid_representation=None) self.assertRaises(ValueError, CodecOptions, uuid_representation=7) self.assertRaises(ValueError, CodecOptions, uuid_representation=2) def test_tzinfo(self): self.assertRaises(TypeError, CodecOptions, tzinfo='pacific') tz = FixedOffset(42, 'forty-two') self.assertRaises(ValueError, CodecOptions, tzinfo=tz) self.assertEqual(tz, CodecOptions(tz_aware=True, tzinfo=tz).tzinfo) def test_codec_options_repr(self): r = ("CodecOptions(document_class=dict, tz_aware=False, " "uuid_representation=PYTHON_LEGACY, " "unicode_decode_error_handler='strict', " "tzinfo=None)") self.assertEqual(r, repr(CodecOptions())) def test_decode_all_defaults(self): # Test decode_all()'s default document_class is dict and tz_aware is # False. The default uuid_representation is PYTHON_LEGACY but this # decodes same as STANDARD, so all this test proves about UUID decoding # is that it's not CSHARP_LEGACY or JAVA_LEGACY. doc = {'sub_document': {}, 'uuid': uuid.uuid4(), 'dt': datetime.datetime.utcnow()} decoded = bson.decode_all(bson.BSON.encode(doc))[0] self.assertIsInstance(decoded['sub_document'], dict) self.assertEqual(decoded['uuid'], doc['uuid']) self.assertIsNone(decoded['dt'].tzinfo) def test_unicode_decode_error_handler(self): enc = BSON.encode({"keystr": "foobar"}) # Test handling of bad key value. invalid_key = BSON(enc[:7] + b'\xe9' + enc[8:]) replaced_key = b'ke\xe9str'.decode('utf-8', 'replace') ignored_key = b'ke\xe9str'.decode('utf-8', 'ignore') dec = BSON.decode(invalid_key, CodecOptions( unicode_decode_error_handler="replace")) self.assertEqual(dec, {replaced_key: u"foobar"}) dec = BSON.decode(invalid_key, CodecOptions( unicode_decode_error_handler="ignore")) self.assertEqual(dec, {ignored_key: u"foobar"}) self.assertRaises(InvalidBSON, BSON.decode, invalid_key, CodecOptions( unicode_decode_error_handler="strict")) self.assertRaises(InvalidBSON, BSON.decode, invalid_key, CodecOptions()) self.assertRaises(InvalidBSON, BSON.decode, invalid_key) # Test handing of bad string value. invalid_val = BSON(enc[:18] + b'\xe9' + enc[19:]) replaced_val = b'fo\xe9bar'.decode('utf-8', 'replace') ignored_val = b'fo\xe9bar'.decode('utf-8', 'ignore') dec = BSON.decode(invalid_val, CodecOptions( unicode_decode_error_handler="replace")) self.assertEqual(dec, {u"keystr": replaced_val}) dec = BSON.decode(invalid_val, CodecOptions( unicode_decode_error_handler="ignore")) self.assertEqual(dec, {u"keystr": ignored_val}) self.assertRaises(InvalidBSON, BSON.decode, invalid_val, CodecOptions( unicode_decode_error_handler="strict")) self.assertRaises(InvalidBSON, BSON.decode, invalid_val, CodecOptions()) self.assertRaises(InvalidBSON, BSON.decode, invalid_val) # Test handing bad key + bad value. invalid_both = BSON( enc[:7] + b'\xe9' + enc[8:18] + b'\xe9' + enc[19:]) dec = BSON.decode(invalid_both, CodecOptions( unicode_decode_error_handler="replace")) self.assertEqual(dec, {replaced_key: replaced_val}) dec = BSON.decode(invalid_both, CodecOptions( unicode_decode_error_handler="ignore")) self.assertEqual(dec, {ignored_key: ignored_val}) self.assertRaises(InvalidBSON, BSON.decode, invalid_both, CodecOptions( unicode_decode_error_handler="strict")) self.assertRaises(InvalidBSON, BSON.decode, invalid_both, CodecOptions()) self.assertRaises(InvalidBSON, BSON.decode, invalid_both) # Test handling bad error mode. dec = BSON.decode(enc, CodecOptions( unicode_decode_error_handler="junk")) self.assertEqual(dec, {"keystr": "foobar"}) self.assertRaises(InvalidBSON, BSON.decode, invalid_both, CodecOptions(unicode_decode_error_handler="junk")) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_objectid.py0000644000076600000240000001511013245621354017611 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the objectid module.""" import datetime import pickle import sys sys.path[0:0] = [""] from bson.errors import InvalidId from bson.objectid import ObjectId from bson.py3compat import PY3, _unicode from bson.tz_util import (FixedOffset, utc) from test import SkipTest, unittest from test.utils import oid_generated_on_client def oid(x): return ObjectId() class TestObjectId(unittest.TestCase): def test_creation(self): self.assertRaises(TypeError, ObjectId, 4) self.assertRaises(TypeError, ObjectId, 175.0) self.assertRaises(TypeError, ObjectId, {"test": 4}) self.assertRaises(TypeError, ObjectId, ["something"]) self.assertRaises(InvalidId, ObjectId, "") self.assertRaises(InvalidId, ObjectId, "12345678901") self.assertRaises(InvalidId, ObjectId, "1234567890123") self.assertTrue(ObjectId()) self.assertTrue(ObjectId(b"123456789012")) a = ObjectId() self.assertTrue(ObjectId(a)) def test_unicode(self): a = ObjectId() self.assertEqual(a, ObjectId(_unicode(a))) self.assertEqual(ObjectId("123456789012123456789012"), ObjectId(u"123456789012123456789012")) self.assertRaises(InvalidId, ObjectId, u"hello") def test_from_hex(self): ObjectId("123456789012123456789012") self.assertRaises(InvalidId, ObjectId, "123456789012123456789G12") self.assertRaises(InvalidId, ObjectId, u"123456789012123456789G12") def test_repr_str(self): self.assertEqual(repr(ObjectId("1234567890abcdef12345678")), "ObjectId('1234567890abcdef12345678')") self.assertEqual(str(ObjectId("1234567890abcdef12345678")), "1234567890abcdef12345678") self.assertEqual(str(ObjectId(b"123456789012")), "313233343536373839303132") self.assertEqual(ObjectId("1234567890abcdef12345678").binary, b'\x124Vx\x90\xab\xcd\xef\x124Vx') self.assertEqual(str(ObjectId(b'\x124Vx\x90\xab\xcd\xef\x124Vx')), "1234567890abcdef12345678") def test_equality(self): a = ObjectId() self.assertEqual(a, ObjectId(a)) self.assertEqual(ObjectId(b"123456789012"), ObjectId(b"123456789012")) self.assertNotEqual(ObjectId(), ObjectId()) self.assertNotEqual(ObjectId(b"123456789012"), b"123456789012") # Explicitly test inequality self.assertFalse(a != ObjectId(a)) self.assertFalse(ObjectId(b"123456789012") != ObjectId(b"123456789012")) def test_binary_str_equivalence(self): a = ObjectId() self.assertEqual(a, ObjectId(a.binary)) self.assertEqual(a, ObjectId(str(a))) def test_pid(self): self.assertTrue(oid_generated_on_client(ObjectId())) def test_generation_time(self): d1 = datetime.datetime.utcnow() d2 = ObjectId().generation_time self.assertEqual(utc, d2.tzinfo) d2 = d2.replace(tzinfo=None) self.assertTrue(d2 - d1 < datetime.timedelta(seconds=2)) def test_from_datetime(self): if 'PyPy 1.8.0' in sys.version: # See https://bugs.pypy.org/issue1092 raise SkipTest("datetime.timedelta is broken in pypy 1.8.0") d = datetime.datetime.utcnow() d = d - datetime.timedelta(microseconds=d.microsecond) oid = ObjectId.from_datetime(d) self.assertEqual(d, oid.generation_time.replace(tzinfo=None)) self.assertEqual("0" * 16, str(oid)[8:]) aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc) oid = ObjectId.from_datetime(aware) self.assertEqual(as_utc, oid.generation_time) def test_pickling(self): orig = ObjectId() for protocol in [0, 1, 2, -1]: pkl = pickle.dumps(orig, protocol=protocol) self.assertEqual(orig, pickle.loads(pkl)) def test_pickle_backwards_compatability(self): # This string was generated by pickling an ObjectId in pymongo # version 1.9 pickled_with_1_9 = ( b"ccopy_reg\n_reconstructor\np0\n" b"(cbson.objectid\nObjectId\np1\nc__builtin__\n" b"object\np2\nNtp3\nRp4\n" b"(dp5\nS'_ObjectId__id'\np6\n" b"S'M\\x9afV\\x13v\\xc0\\x0b\\x88\\x00\\x00\\x00'\np7\nsb.") # We also test against a hardcoded "New" pickle format so that we # make sure we're backward compatible with the current version in # the future as well. pickled_with_1_10 = ( b"ccopy_reg\n_reconstructor\np0\n" b"(cbson.objectid\nObjectId\np1\nc__builtin__\n" b"object\np2\nNtp3\nRp4\n" b"S'M\\x9afV\\x13v\\xc0\\x0b\\x88\\x00\\x00\\x00'\np5\nb.") if PY3: # Have to load using 'latin-1' since these were pickled in python2.x. oid_1_9 = pickle.loads(pickled_with_1_9, encoding='latin-1') oid_1_10 = pickle.loads(pickled_with_1_10, encoding='latin-1') else: oid_1_9 = pickle.loads(pickled_with_1_9) oid_1_10 = pickle.loads(pickled_with_1_10) self.assertEqual(oid_1_9, ObjectId("4d9a66561376c00b88000000")) self.assertEqual(oid_1_9, oid_1_10) def test_is_valid(self): self.assertFalse(ObjectId.is_valid(None)) self.assertFalse(ObjectId.is_valid(4)) self.assertFalse(ObjectId.is_valid(175.0)) self.assertFalse(ObjectId.is_valid({"test": 4})) self.assertFalse(ObjectId.is_valid(["something"])) self.assertFalse(ObjectId.is_valid("")) self.assertFalse(ObjectId.is_valid("12345678901")) self.assertFalse(ObjectId.is_valid("1234567890123")) self.assertTrue(ObjectId.is_valid(b"123456789012")) self.assertTrue(ObjectId.is_valid("123456789012123456789012")) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_database.py0000644000076600000240000011120213245621354017571 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the database module.""" import datetime import re import sys import warnings sys.path[0:0] = [""] from bson.code import Code from bson.codec_options import CodecOptions from bson.int64 import Int64 from bson.regex import Regex from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import string_type, text_type, PY3 from bson.son import SON from pymongo import (ALL, auth, OFF, SLOW_ONLY, helpers) from pymongo.collection import Collection from pymongo.database import Database from pymongo.errors import (CollectionInvalid, ConfigurationError, ExecutionTimeout, InvalidName, OperationFailure, WriteConcernError) from pymongo.mongo_client import MongoClient from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference from pymongo.write_concern import WriteConcern from test import (client_context, SkipTest, unittest, IntegrationTest) from test.utils import (ignore_deprecations, remove_all_users, rs_or_single_client_noauth, rs_or_single_client, server_started_with_auth, IMPOSSIBLE_WRITE_CONCERN) if PY3: long = int class TestDatabaseNoConnect(unittest.TestCase): """Test Database features on a client that does not connect. """ @classmethod def setUpClass(cls): cls.client = MongoClient(connect=False) def test_name(self): self.assertRaises(TypeError, Database, self.client, 4) self.assertRaises(InvalidName, Database, self.client, "my db") self.assertRaises(InvalidName, Database, self.client, 'my"db') self.assertRaises(InvalidName, Database, self.client, "my\x00db") self.assertRaises(InvalidName, Database, self.client, u"my\u0000db") self.assertEqual("name", Database(self.client, "name").name) def test_get_collection(self): codec_options = CodecOptions(tz_aware=True) write_concern = WriteConcern(w=2, j=True) read_concern = ReadConcern('majority') coll = self.client.pymongo_test.get_collection( 'foo', codec_options, ReadPreference.SECONDARY, write_concern, read_concern) self.assertEqual('foo', coll.name) self.assertEqual(codec_options, coll.codec_options) self.assertEqual(ReadPreference.SECONDARY, coll.read_preference) self.assertEqual(write_concern, coll.write_concern) self.assertEqual(read_concern, coll.read_concern) def test_getattr(self): db = self.client.pymongo_test self.assertTrue(isinstance(db['_does_not_exist'], Collection)) with self.assertRaises(AttributeError) as context: db._does_not_exist # Message should be: "AttributeError: Database has no attribute # '_does_not_exist'. To access the _does_not_exist collection, # use database['_does_not_exist']". self.assertIn("has no attribute '_does_not_exist'", str(context.exception)) def test_iteration(self): self.assertRaises(TypeError, next, self.client.pymongo_test) class TestDatabase(IntegrationTest): def test_equality(self): self.assertNotEqual(Database(self.client, "test"), Database(self.client, "mike")) self.assertEqual(Database(self.client, "test"), Database(self.client, "test")) # Explicitly test inequality self.assertFalse(Database(self.client, "test") != Database(self.client, "test")) def test_get_coll(self): db = Database(self.client, "pymongo_test") self.assertEqual(db.test, db["test"]) self.assertEqual(db.test, Collection(db, "test")) self.assertNotEqual(db.test, Collection(db, "mike")) self.assertEqual(db.test.mike, db["test.mike"]) def test_repr(self): self.assertEqual(repr(Database(self.client, "pymongo_test")), "Database(%r, %s)" % (self.client, repr(u"pymongo_test"))) def test_create_collection(self): db = Database(self.client, "pymongo_test") db.test.insert_one({"hello": "world"}) self.assertRaises(CollectionInvalid, db.create_collection, "test") db.drop_collection("test") self.assertRaises(TypeError, db.create_collection, 5) self.assertRaises(TypeError, db.create_collection, None) self.assertRaises(InvalidName, db.create_collection, "coll..ection") test = db.create_collection("test") self.assertTrue(u"test" in db.collection_names()) test.insert_one({"hello": u"world"}) self.assertEqual(db.test.find_one()["hello"], "world") db.drop_collection("test.foo") db.create_collection("test.foo") self.assertTrue(u"test.foo" in db.collection_names()) self.assertRaises(CollectionInvalid, db.create_collection, "test.foo") def _test_collection_names(self, meth, test_no_system): db = Database(self.client, "pymongo_test") db.test.insert_one({"dummy": u"object"}) db.test.mike.insert_one({"dummy": u"object"}) colls = getattr(db, meth)() self.assertTrue("test" in colls) self.assertTrue("test.mike" in colls) for coll in colls: self.assertTrue("$" not in coll) if test_no_system: db.systemcoll.test.insert_one({}) no_system_collections = getattr( db, meth)(include_system_collections=False) for coll in no_system_collections: self.assertTrue(not coll.startswith("system.")) self.assertIn("systemcoll.test", no_system_collections) # Force more than one batch. db = self.client.many_collections for i in range(101): db["coll" + str(i)].insert_one({}) # No Error try: getattr(db, meth)() finally: self.client.drop_database("many_collections") def test_collection_names(self): self._test_collection_names('collection_names', True) def test_list_collection_names(self): self._test_collection_names('list_collection_names', False) def test_list_collections(self): self.client.drop_database("pymongo_test") db = Database(self.client, "pymongo_test") db.test.insert_one({"dummy": u"object"}) db.test.mike.insert_one({"dummy": u"object"}) results = db.list_collections() colls = [result["name"] for result in results] # All the collections present. self.assertTrue("test" in colls) self.assertTrue("test.mike" in colls) # No collection containing a '$'. for coll in colls: self.assertTrue("$" not in coll) # Duplicate check. coll_cnt = {} for coll in colls: try: # Found duplicate. coll_cnt[coll] += 1 self.assertTrue(False) except KeyError: coll_cnt[coll] = 1 coll_cnt = {} # Checking if is there any collection which don't exists. if (len(set(colls) - set(["test","test.mike"])) == 0 or len(set(colls) - set(["test","test.mike","system.indexes"])) == 0): self.assertTrue(True) else: self.assertTrue(False) colls = db.list_collections(filter={"name": {"$regex": "^test$"}}) self.assertEqual(1, len(list(colls))) colls = db.list_collections(filter={"name": {"$regex": "^test.mike$"}}) self.assertEqual(1, len(list(colls))) db.drop_collection("test") db.create_collection("test", capped=True, size=4096) results = db.list_collections(filter={'options.capped': True}) colls = [result["name"] for result in results] # Checking only capped collections are present self.assertTrue("test" in colls) self.assertFalse("test.mike" in colls) # No collection containing a '$'. for coll in colls: self.assertTrue("$" not in coll) # Duplicate check. coll_cnt = {} for coll in colls: try: # Found duplicate. coll_cnt[coll] += 1 self.assertTrue(False) except KeyError: coll_cnt[coll] = 1 coll_cnt = {} # Checking if is there any collection which don't exists. if (len(set(colls) - set(["test"])) == 0 or len(set(colls) - set(["test","system.indexes"])) == 0): self.assertTrue(True) else: self.assertTrue(False) self.client.drop_database("pymongo_test") def test_collection_names_single_socket(self): # Test that Database.collection_names only requires one socket. client = rs_or_single_client(maxPoolSize=1) client.drop_database('test_collection_names_single_socket') db = client.test_collection_names_single_socket for i in range(200): db.create_collection(str(i)) db.collection_names() # Must not hang. client.drop_database('test_collection_names_single_socket') def test_drop_collection(self): db = Database(self.client, "pymongo_test") self.assertRaises(TypeError, db.drop_collection, 5) self.assertRaises(TypeError, db.drop_collection, None) db.test.insert_one({"dummy": u"object"}) self.assertTrue("test" in db.collection_names()) db.drop_collection("test") self.assertFalse("test" in db.collection_names()) db.test.insert_one({"dummy": u"object"}) self.assertTrue("test" in db.collection_names()) db.drop_collection(u"test") self.assertFalse("test" in db.collection_names()) db.test.insert_one({"dummy": u"object"}) self.assertTrue("test" in db.collection_names()) db.drop_collection(db.test) self.assertFalse("test" in db.collection_names()) db.test.insert_one({"dummy": u"object"}) self.assertTrue("test" in db.collection_names()) db.test.drop() self.assertFalse("test" in db.collection_names()) db.test.drop() db.drop_collection(db.test.doesnotexist) if client_context.version.at_least(3, 3, 9) and client_context.is_rs: db_wc = Database(self.client, 'pymongo_test', write_concern=IMPOSSIBLE_WRITE_CONCERN) with self.assertRaises(WriteConcernError): db_wc.drop_collection('test') def test_validate_collection(self): db = self.client.pymongo_test self.assertRaises(TypeError, db.validate_collection, 5) self.assertRaises(TypeError, db.validate_collection, None) db.test.insert_one({"dummy": u"object"}) self.assertRaises(OperationFailure, db.validate_collection, "test.doesnotexist") self.assertRaises(OperationFailure, db.validate_collection, db.test.doesnotexist) self.assertTrue(db.validate_collection("test")) self.assertTrue(db.validate_collection(db.test)) self.assertTrue(db.validate_collection(db.test, full=True)) self.assertTrue(db.validate_collection(db.test, scandata=True)) self.assertTrue(db.validate_collection(db.test, scandata=True, full=True)) self.assertTrue(db.validate_collection(db.test, True, True)) @client_context.require_no_mongos def test_profiling_levels(self): db = self.client.pymongo_test self.assertEqual(db.profiling_level(), OFF) # default self.assertRaises(ValueError, db.set_profiling_level, 5.5) self.assertRaises(ValueError, db.set_profiling_level, None) self.assertRaises(ValueError, db.set_profiling_level, -1) self.assertRaises(TypeError, db.set_profiling_level, SLOW_ONLY, 5.5) self.assertRaises(TypeError, db.set_profiling_level, SLOW_ONLY, '1') db.set_profiling_level(SLOW_ONLY) self.assertEqual(db.profiling_level(), SLOW_ONLY) db.set_profiling_level(ALL) self.assertEqual(db.profiling_level(), ALL) db.set_profiling_level(OFF) self.assertEqual(db.profiling_level(), OFF) db.set_profiling_level(SLOW_ONLY, 50) self.assertEqual(50, db.command("profile", -1)['slowms']) db.set_profiling_level(ALL, -1) self.assertEqual(-1, db.command("profile", -1)['slowms']) db.set_profiling_level(OFF, 100) # back to default self.assertEqual(100, db.command("profile", -1)['slowms']) @client_context.require_no_mongos def test_profiling_info(self): db = self.client.pymongo_test db.system.profile.drop() db.set_profiling_level(ALL) db.test.find_one() db.set_profiling_level(OFF) info = db.profiling_info() self.assertTrue(isinstance(info, list)) # Check if we're going to fail because of SERVER-4754, in which # profiling info isn't collected if mongod was started with --auth if server_started_with_auth(self.client): raise SkipTest( "We need SERVER-4754 fixed for the rest of this test to pass" ) self.assertTrue(len(info) >= 1) # These basically clue us in to server changes. self.assertTrue(isinstance(info[0]['responseLength'], int)) self.assertTrue(isinstance(info[0]['millis'], int)) self.assertTrue(isinstance(info[0]['client'], string_type)) self.assertTrue(isinstance(info[0]['user'], string_type)) self.assertTrue(isinstance(info[0]['ns'], string_type)) self.assertTrue(isinstance(info[0]['op'], string_type)) self.assertTrue(isinstance(info[0]["ts"], datetime.datetime)) @client_context.require_no_mongos def test_errors(self): with ignore_deprecations(): # We must call getlasterror, etc. on same socket as last operation. db = rs_or_single_client(maxPoolSize=1).pymongo_test db.reset_error_history() self.assertEqual(None, db.error()) self.assertEqual(None, db.previous_error()) db.test.insert_one({"_id": 1}) unacked = db.test.with_options(write_concern=WriteConcern(w=0)) unacked.insert_one({"_id": 1}) self.assertTrue(db.error()) self.assertTrue(db.previous_error()) unacked.insert_one({"_id": 1}) self.assertTrue(db.error()) prev_error = db.previous_error() self.assertEqual(prev_error["nPrev"], 1) del prev_error["nPrev"] prev_error.pop("lastOp", None) error = db.error() error.pop("lastOp", None) # getLastError includes "connectionId" in recent # server versions, getPrevError does not. error.pop("connectionId", None) self.assertEqual(error, prev_error) db.test.find_one() self.assertEqual(None, db.error()) self.assertTrue(db.previous_error()) self.assertEqual(db.previous_error()["nPrev"], 2) db.reset_error_history() self.assertEqual(None, db.error()) self.assertEqual(None, db.previous_error()) def test_command(self): self.maxDiff = None db = self.client.admin first = db.command("buildinfo") second = db.command({"buildinfo": 1}) third = db.command("buildinfo", 1) # The logicalTime and operationTime fields were introduced in MongoDB # 3.5. Their value can change from one command call to the next. for doc in (first, second, third): doc.pop("logicalTime", None) doc.pop("operationTime", None) self.assertEqual(first, second) self.assertEqual(second, third) # We use 'aggregate' as our example command, since it's an easy way to # retrieve a BSON regex from a collection using a command. But until # MongoDB 2.3.2, aggregation turned regexes into strings: SERVER-6470. # Note: MongoDB 3.5.2 requires the 'cursor' or 'explain' option for # aggregate. @client_context.require_version_max(3, 5, 0) def test_command_with_regex(self): db = self.client.pymongo_test db.test.drop() db.test.insert_one({'r': re.compile('.*')}) db.test.insert_one({'r': Regex('.*')}) result = db.command('aggregate', 'test', pipeline=[]) for doc in result['result']: self.assertTrue(isinstance(doc['r'], Regex)) def test_password_digest(self): self.assertRaises(TypeError, auth._password_digest, 5) self.assertRaises(TypeError, auth._password_digest, True) self.assertRaises(TypeError, auth._password_digest, None) self.assertTrue(isinstance(auth._password_digest("mike", "password"), text_type)) self.assertEqual(auth._password_digest("mike", "password"), u"cd7e45b3b2767dc2fa9b6b548457ed00") self.assertEqual(auth._password_digest("mike", "password"), auth._password_digest(u"mike", u"password")) self.assertEqual(auth._password_digest("Gustave", u"Dor\xe9"), u"81e0e2364499209f466e75926a162d73") @client_context.require_auth def test_authenticate_add_remove_user(self): # "self.client" is logged in as root. auth_db = self.client.pymongo_test def check_auth(username, password): c = rs_or_single_client_noauth( username=username, password=password, authSource="pymongo_test") c.pymongo_test.collection.find_one() # Configuration errors self.assertRaises(ValueError, auth_db.add_user, "user", '') self.assertRaises(TypeError, auth_db.add_user, "user", 'password', 15) self.assertRaises(TypeError, auth_db.add_user, "user", 'password', 'True') self.assertRaises(ConfigurationError, auth_db.add_user, "user", 'password', True, roles=['read']) with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) self.assertRaises(DeprecationWarning, auth_db.add_user, "user", "password") self.assertRaises(DeprecationWarning, auth_db.add_user, "user", "password", True) with ignore_deprecations(): self.assertRaises(ConfigurationError, auth_db.add_user, "user", "password", digestPassword=True) extra = {} if client_context.version.at_least(3, 7, 2): extra['mechanisms'] = ['SCRAM-SHA-1'] # Add / authenticate / remove auth_db.add_user("mike", "password", roles=["read"], **extra) self.addCleanup(remove_all_users, auth_db) self.assertRaises(TypeError, check_auth, 5, "password") self.assertRaises(TypeError, check_auth, "mike", 5) self.assertRaises(OperationFailure, check_auth, "mike", "not a real password") self.assertRaises(OperationFailure, check_auth, "faker", "password") check_auth("mike", "password") # Unicode name and password. check_auth(u"mike", u"password") auth_db.remove_user("mike") self.assertRaises(OperationFailure, check_auth, "mike", "password") # Add / authenticate / change password self.assertRaises(OperationFailure, check_auth, "Gustave", u"Dor\xe9") auth_db.add_user("Gustave", u"Dor\xe9", roles=["read"], **extra) check_auth("Gustave", u"Dor\xe9") # Change password. auth_db.add_user("Gustave", "password", roles=["read"], **extra) self.assertRaises(OperationFailure, check_auth, "Gustave", u"Dor\xe9") check_auth("Gustave", u"password") @client_context.require_auth @ignore_deprecations def test_make_user_readonly(self): extra = {} if client_context.version.at_least(3, 7, 2): extra['mechanisms'] = ['SCRAM-SHA-1'] # "self.client" is logged in as root. auth_db = self.client.pymongo_test # Make a read-write user. auth_db.add_user('jesse', 'pw', **extra) self.addCleanup(remove_all_users, auth_db) # Check that we're read-write by default. c = rs_or_single_client_noauth(username='jesse', password='pw', authSource='pymongo_test') c.pymongo_test.collection.insert_one({}) # Make the user read-only. auth_db.add_user('jesse', 'pw', read_only=True, **extra) c = rs_or_single_client_noauth(username='jesse', password='pw', authSource='pymongo_test') self.assertRaises(OperationFailure, c.pymongo_test.collection.insert_one, {}) @client_context.require_auth @ignore_deprecations def test_default_roles(self): extra = {} if client_context.version.at_least(3, 7, 2): extra['mechanisms'] = ['SCRAM-SHA-1'] # "self.client" is logged in as root. auth_admin = self.client.admin auth_admin.add_user('test_default_roles', 'pass', **extra) self.addCleanup(client_context.drop_user, 'admin', 'test_default_roles') info = auth_admin.command( 'usersInfo', 'test_default_roles')['users'][0] self.assertEqual("root", info['roles'][0]['role']) # Read only "admin" user auth_admin.add_user('ro-admin', 'pass', read_only=True, **extra) self.addCleanup(client_context.drop_user, 'admin', 'ro-admin') info = auth_admin.command('usersInfo', 'ro-admin')['users'][0] self.assertEqual("readAnyDatabase", info['roles'][0]['role']) # "Non-admin" user auth_db = self.client.pymongo_test auth_db.add_user('user', 'pass', **extra) self.addCleanup(remove_all_users, auth_db) info = auth_db.command('usersInfo', 'user')['users'][0] self.assertEqual("dbOwner", info['roles'][0]['role']) # Read only "Non-admin" user auth_db.add_user('ro-user', 'pass', read_only=True, **extra) info = auth_db.command('usersInfo', 'ro-user')['users'][0] self.assertEqual("read", info['roles'][0]['role']) @client_context.require_auth @ignore_deprecations def test_new_user_cmds(self): extra = {} if client_context.version.at_least(3, 7, 2): extra['mechanisms'] = ['SCRAM-SHA-1'] # "self.client" is logged in as root. auth_db = self.client.pymongo_test auth_db.add_user("amalia", "password", roles=["userAdmin"], **extra) self.addCleanup(client_context.drop_user, "pymongo_test", "amalia") db = rs_or_single_client_noauth(username="amalia", password="password", authSource="pymongo_test").pymongo_test # This tests the ability to update user attributes. db.add_user("amalia", "new_password", customData={"secret": "koalas"}, **extra) user_info = db.command("usersInfo", "amalia") self.assertTrue(user_info["users"]) amalia_user = user_info["users"][0] self.assertEqual(amalia_user["user"], "amalia") self.assertEqual(amalia_user["customData"], {"secret": "koalas"}) @client_context.require_auth @ignore_deprecations def test_authenticate_multiple(self): extra = {} if client_context.version.at_least(3, 7, 2): extra['mechanisms'] = ['SCRAM-SHA-1'] # "self.client" is logged in as root. self.client.drop_database("pymongo_test") self.client.drop_database("pymongo_test1") admin_db_auth = self.client.admin users_db_auth = self.client.pymongo_test # Non-root client. client = rs_or_single_client_noauth() admin_db = client.admin users_db = client.pymongo_test other_db = client.pymongo_test1 self.assertRaises(OperationFailure, users_db.test.find_one) admin_db_auth.add_user( 'ro-admin', 'pass', roles=["userAdmin", "readAnyDatabase"], **extra) self.addCleanup(client_context.drop_user, 'admin', 'ro-admin') users_db_auth.add_user( 'user', 'pass', roles=["userAdmin", "readWrite"], **extra) self.addCleanup(remove_all_users, users_db_auth) # Regular user should be able to query its own db, but # no other. users_db.authenticate('user', 'pass') self.assertEqual(0, users_db.test.count()) self.assertRaises(OperationFailure, other_db.test.find_one) # Admin read-only user should be able to query any db, # but not write. admin_db.authenticate('ro-admin', 'pass') self.assertEqual(None, other_db.test.find_one()) self.assertRaises(OperationFailure, other_db.test.insert_one, {}) # Close all sockets. client.close() # We should still be able to write to the regular user's db. self.assertTrue(users_db.test.delete_many({})) # And read from other dbs... self.assertEqual(0, other_db.test.count()) # But still not write to other dbs. self.assertRaises(OperationFailure, other_db.test.insert_one, {}) def test_id_ordering(self): # PyMongo attempts to have _id show up first # when you iterate key/value pairs in a document. # This isn't reliable since python dicts don't # guarantee any particular order. This will never # work right in Jython or any Python or environment # with hash randomization enabled (e.g. tox). db = self.client.pymongo_test db.test.drop() db.test.insert_one(SON([("hello", "world"), ("_id", 5)])) db = self.client.get_database( "pymongo_test", codec_options=CodecOptions(document_class=SON)) cursor = db.test.find() for x in cursor: for (k, v) in x.items(): self.assertEqual(k, "_id") break def test_deref(self): db = self.client.pymongo_test db.test.drop() self.assertRaises(TypeError, db.dereference, 5) self.assertRaises(TypeError, db.dereference, "hello") self.assertRaises(TypeError, db.dereference, None) self.assertEqual(None, db.dereference(DBRef("test", ObjectId()))) obj = {"x": True} key = db.test.insert_one(obj).inserted_id self.assertEqual(obj, db.dereference(DBRef("test", key))) self.assertEqual(obj, db.dereference(DBRef("test", key, "pymongo_test"))) self.assertRaises(ValueError, db.dereference, DBRef("test", key, "foo")) self.assertEqual(None, db.dereference(DBRef("test", 4))) obj = {"_id": 4} db.test.insert_one(obj) self.assertEqual(obj, db.dereference(DBRef("test", 4))) def test_deref_kwargs(self): db = self.client.pymongo_test db.test.drop() db.test.insert_one({"_id": 4, "foo": "bar"}) db = self.client.get_database( "pymongo_test", codec_options=CodecOptions(document_class=SON)) self.assertEqual(SON([("foo", "bar")]), db.dereference(DBRef("test", 4), projection={"_id": False})) @client_context.require_no_auth def test_eval(self): db = self.client.pymongo_test db.test.drop() with ignore_deprecations(): self.assertRaises(TypeError, db.eval, None) self.assertRaises(TypeError, db.eval, 5) self.assertRaises(TypeError, db.eval, []) self.assertEqual(3, db.eval("function (x) {return x;}", 3)) self.assertEqual(3, db.eval(u"function (x) {return x;}", 3)) self.assertEqual(None, db.eval("function (x) {db.test.save({y:x});}", 5)) self.assertEqual(db.test.find_one()["y"], 5) self.assertEqual(5, db.eval("function (x, y) {return x + y;}", 2, 3)) self.assertEqual(5, db.eval("function () {return 5;}")) self.assertEqual(5, db.eval("2 + 3;")) self.assertEqual(5, db.eval(Code("2 + 3;"))) self.assertRaises(OperationFailure, db.eval, Code("return i;")) self.assertEqual(2, db.eval(Code("return i;", {"i": 2}))) self.assertEqual(5, db.eval(Code("i + 3;", {"i": 2}))) self.assertRaises(OperationFailure, db.eval, "5 ++ 5;") # TODO some of these tests belong in the collection level testing. def test_insert_find_one(self): db = self.client.pymongo_test db.test.drop() a_doc = SON({"hello": u"world"}) a_key = db.test.insert_one(a_doc).inserted_id self.assertTrue(isinstance(a_doc["_id"], ObjectId)) self.assertEqual(a_doc["_id"], a_key) self.assertEqual(a_doc, db.test.find_one({"_id": a_doc["_id"]})) self.assertEqual(a_doc, db.test.find_one(a_key)) self.assertEqual(None, db.test.find_one(ObjectId())) self.assertEqual(a_doc, db.test.find_one({"hello": u"world"})) self.assertEqual(None, db.test.find_one({"hello": u"test"})) b = db.test.find_one() b["hello"] = u"mike" db.test.replace_one({"_id": b["_id"]}, b) self.assertNotEqual(a_doc, db.test.find_one(a_key)) self.assertEqual(b, db.test.find_one(a_key)) self.assertEqual(b, db.test.find_one()) count = 0 for _ in db.test.find(): count += 1 self.assertEqual(count, 1) def test_long(self): db = self.client.pymongo_test db.test.drop() db.test.insert_one({"x": long(9223372036854775807)}) retrieved = db.test.find_one()['x'] self.assertEqual(Int64(9223372036854775807), retrieved) self.assertIsInstance(retrieved, Int64) db.test.delete_many({}) db.test.insert_one({"x": Int64(1)}) retrieved = db.test.find_one()['x'] self.assertEqual(Int64(1), retrieved) self.assertIsInstance(retrieved, Int64) def test_delete(self): db = self.client.pymongo_test db.test.drop() db.test.insert_one({"x": 1}) db.test.insert_one({"x": 2}) db.test.insert_one({"x": 3}) length = 0 for _ in db.test.find(): length += 1 self.assertEqual(length, 3) db.test.delete_one({"x": 1}) length = 0 for _ in db.test.find(): length += 1 self.assertEqual(length, 2) db.test.delete_one(db.test.find_one()) db.test.delete_one(db.test.find_one()) self.assertEqual(db.test.find_one(), None) db.test.insert_one({"x": 1}) db.test.insert_one({"x": 2}) db.test.insert_one({"x": 3}) self.assertTrue(db.test.find_one({"x": 2})) db.test.delete_one({"x": 2}) self.assertFalse(db.test.find_one({"x": 2})) self.assertTrue(db.test.find_one()) db.test.delete_many({}) self.assertFalse(db.test.find_one()) @client_context.require_no_auth def test_system_js(self): db = self.client.pymongo_test db.system.js.delete_many({}) self.assertEqual(0, db.system.js.count()) db.system_js.add = "function(a, b) { return a + b; }" self.assertEqual('add', db.system.js.find_one()['_id']) self.assertEqual(1, db.system.js.count()) self.assertEqual(6, db.system_js.add(1, 5)) del db.system_js.add self.assertEqual(0, db.system.js.count()) db.system_js['add'] = "function(a, b) { return a + b; }" self.assertEqual('add', db.system.js.find_one()['_id']) self.assertEqual(1, db.system.js.count()) self.assertEqual(6, db.system_js['add'](1, 5)) del db.system_js['add'] self.assertEqual(0, db.system.js.count()) self.assertRaises(OperationFailure, db.system_js.add, 1, 5) # TODO right now CodeWScope doesn't work w/ system js # db.system_js.scope = Code("return hello;", {"hello": 8}) # self.assertEqual(8, db.system_js.scope()) self.assertRaises(OperationFailure, db.system_js.non_existant) def test_system_js_list(self): db = self.client.pymongo_test db.system.js.delete_many({}) self.assertEqual([], db.system_js.list()) db.system_js.foo = "function() { return 'blah'; }" self.assertEqual(["foo"], db.system_js.list()) db.system_js.bar = "function() { return 'baz'; }" self.assertEqual(set(["foo", "bar"]), set(db.system_js.list())) del db.system_js.foo self.assertEqual(["bar"], db.system_js.list()) def test_command_response_without_ok(self): # Sometimes (SERVER-10891) the server's response to a badly-formatted # command document will have no 'ok' field. We should raise # OperationFailure instead of KeyError. self.assertRaises(OperationFailure, helpers._check_command_response, {}) try: helpers._check_command_response({'$err': 'foo'}) except OperationFailure as e: self.assertEqual(e.args[0], 'foo') else: self.fail("_check_command_response didn't raise OperationFailure") def test_mongos_response(self): error_document = { 'ok': 0, 'errmsg': 'outer', 'raw': {'shard0/host0,host1': {'ok': 0, 'errmsg': 'inner'}}} with self.assertRaises(OperationFailure) as context: helpers._check_command_response(error_document) self.assertEqual('inner', str(context.exception)) # If a shard has no primary and you run a command like dbstats, which # cannot be run on a secondary, mongos's response includes empty "raw" # errors. See SERVER-15428. error_document = { 'ok': 0, 'errmsg': 'outer', 'raw': {'shard0/host0,host1': {}}} with self.assertRaises(OperationFailure) as context: helpers._check_command_response(error_document) self.assertEqual('outer', str(context.exception)) # Raw error has ok: 0 but no errmsg. Not a known case, but test it. error_document = { 'ok': 0, 'errmsg': 'outer', 'raw': {'shard0/host0,host1': {'ok': 0}}} with self.assertRaises(OperationFailure) as context: helpers._check_command_response(error_document) self.assertEqual('outer', str(context.exception)) @client_context.require_test_commands @client_context.require_no_mongos def test_command_max_time_ms(self): self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") try: db = self.client.pymongo_test db.command('count', 'test') self.assertRaises(ExecutionTimeout, db.command, 'count', 'test', maxTimeMS=1) pipeline = [{'$project': {'name': 1, 'count': 1}}] # Database command helper. db.command('aggregate', 'test', pipeline=pipeline, cursor={}) self.assertRaises(ExecutionTimeout, db.command, 'aggregate', 'test', pipeline=pipeline, cursor={}, maxTimeMS=1) # Collection helper. db.test.aggregate(pipeline=pipeline) self.assertRaises(ExecutionTimeout, db.test.aggregate, pipeline, maxTimeMS=1) finally: self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_common.py0000644000076600000240000001723113245621354017324 0ustar shanestaff00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the pymongo common module.""" import sys import uuid sys.path[0:0] = [""] from bson.binary import UUIDLegacy, PYTHON_LEGACY, STANDARD from bson.code import Code from bson.codec_options import CodecOptions from bson.objectid import ObjectId from pymongo.errors import OperationFailure from pymongo.write_concern import WriteConcern from test import client_context, IntegrationTest from test.utils import connected, rs_or_single_client, single_client @client_context.require_connection def setUpModule(): pass class TestCommon(IntegrationTest): def test_uuid_representation(self): coll = self.db.uuid coll.drop() # Test property self.assertEqual(PYTHON_LEGACY, coll.codec_options.uuid_representation) # Test basic query uu = uuid.uuid4() # Insert as binary subtype 3 coll.insert_one({'uu': uu}) self.assertEqual(uu, coll.find_one({'uu': uu})['uu']) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) self.assertEqual(STANDARD, coll.codec_options.uuid_representation) self.assertEqual(None, coll.find_one({'uu': uu})) self.assertEqual(uu, coll.find_one({'uu': UUIDLegacy(uu)})['uu']) # Test Cursor.count self.assertEqual(0, coll.find({'uu': uu}).count()) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) self.assertEqual(1, coll.find({'uu': uu}).count()) # Test delete coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) coll.delete_one({'uu': uu}) self.assertEqual(1, coll.count()) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) coll.delete_one({'uu': uu}) self.assertEqual(0, coll.count()) # Test update_one coll.insert_one({'_id': uu, 'i': 1}) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) coll.update_one({'_id': uu}, {'$set': {'i': 2}}) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) self.assertEqual(1, coll.find_one({'_id': uu})['i']) coll.update_one({'_id': uu}, {'$set': {'i': 2}}) self.assertEqual(2, coll.find_one({'_id': uu})['i']) # Test Cursor.distinct self.assertEqual([2], coll.find({'_id': uu}).distinct('i')) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) self.assertEqual([], coll.find({'_id': uu}).distinct('i')) # Test findAndModify self.assertEqual(None, coll.find_one_and_update({'_id': uu}, {'$set': {'i': 5}})) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) self.assertEqual(2, coll.find_one_and_update({'_id': uu}, {'$set': {'i': 5}})['i']) self.assertEqual(5, coll.find_one({'_id': uu})['i']) # Test command self.assertEqual(5, self.db.command('findAndModify', 'uuid', update={'$set': {'i': 6}}, query={'_id': uu})['value']['i']) self.assertEqual(6, self.db.command( 'findAndModify', 'uuid', update={'$set': {'i': 7}}, query={'_id': UUIDLegacy(uu)})['value']['i']) # Test (inline)_map_reduce coll.drop() coll.insert_one({"_id": uu, "x": 1, "tags": ["dog", "cat"]}) coll.insert_one({"_id": uuid.uuid4(), "x": 3, "tags": ["mouse", "cat", "dog"]}) map = Code("function () {" " this.tags.forEach(function(z) {" " emit(z, 1);" " });" "}") reduce = Code("function (key, values) {" " var total = 0;" " for (var i = 0; i < values.length; i++) {" " total += values[i];" " }" " return total;" "}") coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) q = {"_id": uu} result = coll.inline_map_reduce(map, reduce, query=q) self.assertEqual([], result) result = coll.map_reduce(map, reduce, "results", query=q) self.assertEqual(0, self.db.results.count()) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) q = {"_id": uu} result = coll.inline_map_reduce(map, reduce, query=q) self.assertEqual(2, len(result)) result = coll.map_reduce(map, reduce, "results", query=q) self.assertEqual(2, self.db.results.count()) self.db.drop_collection("result") coll.drop() def test_write_concern(self): c = rs_or_single_client(connect=False) self.assertEqual(WriteConcern(), c.write_concern) c = rs_or_single_client(connect=False, w=2, wtimeout=1000) wc = WriteConcern(w=2, wtimeout=1000) self.assertEqual(wc, c.write_concern) db = c.pymongo_test self.assertEqual(wc, db.write_concern) coll = db.test self.assertEqual(wc, coll.write_concern) cwc = WriteConcern(j=True) coll = db.get_collection('test', write_concern=cwc) self.assertEqual(cwc, coll.write_concern) self.assertEqual(wc, db.write_concern) def test_mongo_client(self): pair = client_context.pair m = rs_or_single_client(w=0) coll = m.pymongo_test.write_concern_test coll.drop() doc = {"_id": ObjectId()} coll.insert_one(doc) self.assertTrue(coll.insert_one(doc)) coll = coll.with_options(write_concern=WriteConcern(w=1)) self.assertRaises(OperationFailure, coll.insert_one, doc) m = rs_or_single_client() coll = m.pymongo_test.write_concern_test new_coll = coll.with_options(write_concern=WriteConcern(w=0)) self.assertTrue(new_coll.insert_one(doc)) self.assertRaises(OperationFailure, coll.insert_one, doc) m = rs_or_single_client("mongodb://%s/" % (pair,), replicaSet=client_context.replica_set_name) coll = m.pymongo_test.write_concern_test self.assertRaises(OperationFailure, coll.insert_one, doc) m = rs_or_single_client("mongodb://%s/?w=0" % (pair,), replicaSet=client_context.replica_set_name) coll = m.pymongo_test.write_concern_test coll.insert_one(doc) # Equality tests direct = connected(single_client(w=0)) direct2 = connected(single_client("mongodb://%s/?w=0" % (pair,), **self.credentials)) self.assertEqual(direct, direct2) self.assertFalse(direct != direct2) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_read_concern.py0000644000076600000240000001411213245621354020451 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the read_concern module.""" from bson.son import SON from pymongo import monitoring from pymongo.errors import ConfigurationError, OperationFailure from pymongo.read_concern import ReadConcern from test import client_context, PyMongoTestCase from test.utils import single_client, rs_or_single_client, EventListener class TestReadConcern(PyMongoTestCase): @classmethod @client_context.require_connection def setUpClass(cls): cls.listener = EventListener() cls.saved_listeners = monitoring._LISTENERS # Don't use any global subscribers. monitoring._LISTENERS = monitoring._Listeners([], [], [], []) cls.client = single_client(event_listeners=[cls.listener]) cls.db = cls.client.pymongo_test @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners def tearDown(self): self.db.coll.drop() self.listener.results.clear() def test_read_concern(self): rc = ReadConcern() self.assertIsNone(rc.level) self.assertTrue(rc.ok_for_legacy) rc = ReadConcern('majority') self.assertEqual('majority', rc.level) self.assertFalse(rc.ok_for_legacy) rc = ReadConcern('local') self.assertEqual('local', rc.level) self.assertTrue(rc.ok_for_legacy) self.assertRaises(TypeError, ReadConcern, 42) def test_read_concern_uri(self): uri = 'mongodb://%s/?readConcernLevel=majority' % ( client_context.pair,) client = rs_or_single_client(uri, connect=False) self.assertEqual(ReadConcern('majority'), client.read_concern) @client_context.require_version_max(3, 1) def test_invalid_read_concern(self): coll = self.db.get_collection( 'coll', read_concern=ReadConcern('majority')) self.assertRaisesRegexp( ConfigurationError, 'read concern level of majority is not valid ' 'with a max wire version of [0-3]', coll.count) @client_context.require_version_min(3, 1, 9, -1) def test_find_command(self): # readConcern not sent in command if not specified. coll = self.db.coll tuple(coll.find({'field': 'value'})) self.assertNotIn('readConcern', self.listener.results['started'][0].command) self.listener.results.clear() # Explicitly set readConcern to 'local'. coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) tuple(coll.find({'field': 'value'})) self.assertEqualCommand( SON([('find', 'coll'), ('filter', {'field': 'value'}), ('readConcern', {'level': 'local'})]), self.listener.results['started'][0].command) @client_context.require_version_min(3, 1, 9, -1) def test_command_cursor(self): # readConcern not sent in command if not specified. coll = self.db.coll tuple(coll.aggregate([{'$match': {'field': 'value'}}])) self.assertNotIn('readConcern', self.listener.results['started'][0].command) self.listener.results.clear() # Explicitly set readConcern to 'local'. coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) tuple(coll.aggregate([{'$match': {'field': 'value'}}])) self.assertEqual( {'level': 'local'}, self.listener.results['started'][0].command['readConcern']) def test_aggregate_out(self): coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) try: tuple(coll.aggregate([{'$match': {'field': 'value'}}, {'$out': 'output_collection'}])) except OperationFailure: # "ns doesn't exist" pass self.assertNotIn('readConcern', self.listener.results['started'][0].command) def test_map_reduce_out(self): coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) try: tuple(coll.map_reduce('function() { emit(this._id, this.value); }', 'function(key, values) { return 42; }', out='output_collection')) except OperationFailure: # "ns doesn't exist" pass self.assertNotIn('readConcern', self.listener.results['started'][0].command) if client_context.version.at_least(3, 1, 9, -1): self.listener.results.clear() try: tuple(coll.map_reduce( 'function() { emit(this._id, this.value); }', 'function(key, values) { return 42; }', out={'inline': 1})) except OperationFailure: # "ns doesn't exist" pass self.assertEqual( {'level': 'local'}, self.listener.results['started'][0].command['readConcern']) @client_context.require_version_min(3, 1, 9, -1) def test_inline_map_reduce(self): coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) try: tuple(coll.inline_map_reduce( 'function() { emit(this._id, this.value); }', 'function(key, values) { return 42; }')) except OperationFailure: # "ns doesn't exist" pass self.assertEqual( {'level': 'local'}, self.listener.results['started'][0].command['readConcern']) pymongo-3.6.1/test/version.py0000644000076600000240000000566213245617773016501 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Some tools for running tests based on MongoDB server version.""" class Version(tuple): def __new__(cls, *version): padded_version = cls._padded(version, 4) return super(Version, cls).__new__(cls, tuple(padded_version)) @classmethod def _padded(cls, iter, length, padding=0): l = list(iter) if len(l) < length: for _ in range(length - len(l)): l.append(padding) return l @classmethod def from_string(cls, version_string): mod = 0 bump_patch_level = False if version_string.endswith("+"): version_string = version_string[0:-1] mod = 1 elif version_string.endswith("-pre-"): version_string = version_string[0:-5] mod = -1 elif version_string.endswith("-"): version_string = version_string[0:-1] mod = -1 # Deal with '-rcX' substrings if '-rc' in version_string: version_string = version_string[0:version_string.find('-rc')] mod = -1 # Deal with git describe generated substrings elif '-' in version_string: version_string = version_string[0:version_string.find('-')] mod = -1 bump_patch_level = True version = [int(part) for part in version_string.split(".")] version = cls._padded(version, 3) # Make from_string and from_version_array agree. For example: # MongoDB Enterprise > db.runCommand('buildInfo').versionArray # [ 3, 2, 1, -100 ] # MongoDB Enterprise > db.runCommand('buildInfo').version # 3.2.0-97-g1ef94fe if bump_patch_level: version[-1] += 1 version.append(mod) return Version(*version) @classmethod def from_version_array(cls, version_array): version = list(version_array) if version[-1] < 0: version[-1] = -1 version = cls._padded(version, 3) return Version(*version) @classmethod def from_client(cls, client): info = client.server_info() if 'versionArray' in info: return cls.from_version_array(info['versionArray']) return cls.from_string(info['version']) def at_least(self, *other_version): return self >= Version(*other_version) def __str__(self): return ".".join(map(str, self)) pymongo-3.6.1/test/test_command_monitoring_spec.py0000644000076600000240000002342013245621354022726 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Run the command monitoring spec tests.""" import os import re import sys sys.path[0:0] = [""] import pymongo from bson import json_util from pymongo import monitoring from pymongo.errors import OperationFailure from pymongo.read_preferences import (make_read_preference, read_pref_mode_from_name) from pymongo.write_concern import WriteConcern from test import unittest, client_context from test.utils import single_client, wait_until, EventListener # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'command_monitoring') def camel_to_snake(camel): # Regex to convert CamelCase to snake_case. snake = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', camel) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', snake).lower() class TestAllScenarios(unittest.TestCase): @classmethod @client_context.require_connection def setUpClass(cls): cls.listener = EventListener() cls.saved_listeners = monitoring._LISTENERS monitoring._LISTENERS = monitoring._Listeners([], [], [], []) cls.client = single_client(event_listeners=[cls.listener]) @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners def tearDown(self): self.listener.results.clear() def format_actual_results(results): started = results['started'] succeeded = results['succeeded'] failed = results['failed'] msg = "\nStarted: %r" % (started[0].command if len(started) else None,) msg += "\nSucceeded: %r" % (succeeded[0].reply if len(succeeded) else None,) msg += "\nFailed: %r" % (failed[0].failure if len(failed) else None,) return msg def create_test(scenario_def, test): def run_scenario(self): dbname = scenario_def['database_name'] collname = scenario_def['collection_name'] coll = self.client[dbname][collname] coll.drop() coll.insert_many(scenario_def['data']) self.listener.results.clear() name = camel_to_snake(test['operation']['name']) if 'read_preference' in test['operation']: mode = read_pref_mode_from_name( test['operation']['read_preference']['mode']) coll = coll.with_options( read_preference=make_read_preference(mode, None)) test_args = test['operation']['arguments'] if 'writeConcern' in test_args: concern = test_args.pop('writeConcern') coll = coll.with_options( write_concern=WriteConcern(**concern)) args = {} for arg in test_args: args[camel_to_snake(arg)] = test_args[arg] if name == 'bulk_write': bulk_args = [] for request in args['requests']: opname = next(iter(request)) klass = opname[0:1].upper() + opname[1:] arg = getattr(pymongo, klass)(**request[opname]) bulk_args.append(arg) try: coll.bulk_write(bulk_args, args.get('ordered', True)) except OperationFailure: pass elif name == 'find': if 'sort' in args: args['sort'] = list(args['sort'].items()) for arg in 'skip', 'limit': if arg in args: args[arg] = int(args[arg]) try: # Iterate the cursor. tuple(coll.find(**args)) except OperationFailure: pass # Wait for the killCursors thread to run if necessary. if 'limit' in args and client_context.version[:2] < (3, 1): self.client._kill_cursors_executor.wake() started = self.listener.results['started'] succeeded = self.listener.results['succeeded'] wait_until( lambda: started[-1].command_name == 'killCursors', "publish a start event for killCursors.") wait_until( lambda: succeeded[-1].command_name == 'killCursors', "publish a succeeded event for killCursors.") else: try: getattr(coll, name)(**args) except OperationFailure: pass res = self.listener.results for expectation in test['expectations']: event_type = next(iter(expectation)) if event_type == "command_started_event": event = res['started'][0] if len(res['started']) else None if event is not None: # The tests substitute 42 for any number other than 0. if (event.command_name == 'getMore' and event.command['getMore']): event.command['getMore'] = 42 elif event.command_name == 'killCursors': event.command['cursors'] = [42] elif event_type == "command_succeeded_event": event = ( res['succeeded'].pop(0) if len(res['succeeded']) else None) if event is not None: reply = event.reply # The tests substitute 42 for any number other than 0, # and "" for any error message. if 'writeErrors' in reply: for doc in reply['writeErrors']: # Remove any new fields the server adds. The tests # only have index, code, and errmsg. diff = set(doc) - set(['index', 'code', 'errmsg']) for field in diff: doc.pop(field) doc['code'] = 42 doc['errmsg'] = "" elif 'cursor' in reply: if reply['cursor']['id']: reply['cursor']['id'] = 42 elif event.command_name == 'killCursors': # Make the tests continue to pass when the killCursors # command is actually in use. if 'cursorsKilled' in reply: reply.pop('cursorsKilled') reply['cursorsUnknown'] = [42] # Found succeeded event. Pop related started event. res['started'].pop(0) elif event_type == "command_failed_event": event = res['failed'].pop(0) if len(res['failed']) else None if event is not None: # Found failed event. Pop related started event. res['started'].pop(0) else: self.fail("Unknown event type") if event is None: event_name = event_type.split('_')[1] self.fail( "Expected %s event for %s command. Actual " "results:%s" % ( event_name, expectation[event_type]['command_name'], format_actual_results(res))) for attr, expected in expectation[event_type].items(): actual = getattr(event, attr) if isinstance(expected, dict): for key, val in expected.items(): self.assertEqual(val, actual[key]) else: self.assertEqual(actual, expected) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): dirname = os.path.split(dirpath)[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json_util.loads(scenario_stream.read()) assert bool(scenario_def.get('tests')), "tests cannot be empty" # Construct test from scenario. for test in scenario_def['tests']: new_test = create_test(scenario_def, test) if "ignore_if_server_version_greater_than" in test: version = test["ignore_if_server_version_greater_than"] ver = tuple(int(elt) for elt in version.split('.')) new_test = client_context.require_version_max(*ver)( new_test) if "ignore_if_server_version_less_than" in test: version = test["ignore_if_server_version_less_than"] ver = tuple(int(elt) for elt in version.split('.')) new_test = client_context.require_version_min(*ver)( new_test) if "ignore_if_topology_type" in test: types = set(test["ignore_if_topology_type"]) if "sharded" in types: new_test = client_context.require_no_mongos(None)( new_test) test_name = 'test_%s_%s_%s' % ( dirname, os.path.splitext(filename)[0], str(test['description'].replace(" ", "_"))) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/qcheck.py0000644000076600000240000001676413245621354016245 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import random import traceback import datetime import re import sys sys.path[0:0] = [""] from bson.binary import Binary from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import MAXSIZE, PY3, iteritems from bson.son import SON if PY3: unichr = chr gen_target = 100 reduction_attempts = 10 examples = 5 def lift(value): return lambda: value def choose_lifted(generator_list): return lambda: random.choice(generator_list) def my_map(generator, function): return lambda: function(generator()) def choose(list): return lambda: random.choice(list)() def gen_range(start, stop): return lambda: random.randint(start, stop) def gen_int(): max_int = 2147483647 return lambda: random.randint(-max_int - 1, max_int) def gen_float(): return lambda: (random.random() - 0.5) * MAXSIZE def gen_boolean(): return lambda: random.choice([True, False]) def gen_printable_char(): return lambda: chr(random.randint(32, 126)) def gen_printable_string(gen_length): return lambda: "".join(gen_list(gen_printable_char(), gen_length)()) if PY3: def gen_char(set=None): return lambda: bytes([random.randint(0, 255)]) else: def gen_char(set=None): return lambda: chr(random.randint(0, 255)) def gen_string(gen_length): return lambda: b"".join(gen_list(gen_char(), gen_length)()) def gen_unichar(): return lambda: unichr(random.randint(1, 0xFFF)) def gen_unicode(gen_length): return lambda: u"".join([x for x in gen_list(gen_unichar(), gen_length)() if x not in ".$"]) def gen_list(generator, gen_length): return lambda: [generator() for _ in range(gen_length())] def gen_datetime(): return lambda: datetime.datetime(random.randint(1970, 2037), random.randint(1, 12), random.randint(1, 28), random.randint(0, 23), random.randint(0, 59), random.randint(0, 59), random.randint(0, 999) * 1000) def gen_dict(gen_key, gen_value, gen_length): def a_dict(gen_key, gen_value, length): result = {} for _ in range(length): result[gen_key()] = gen_value() return result return lambda: a_dict(gen_key, gen_value, gen_length()) def gen_regexp(gen_length): # TODO our patterns only consist of one letter. # this is because of a bug in CPython's regex equality testing, # which I haven't quite tracked down, so I'm just ignoring it... pattern = lambda: u"".join(gen_list(choose_lifted(u"a"), gen_length)()) def gen_flags(): flags = 0 if random.random() > 0.5: flags = flags | re.IGNORECASE if random.random() > 0.5: flags = flags | re.MULTILINE if random.random() > 0.5: flags = flags | re.VERBOSE return flags return lambda: re.compile(pattern(), gen_flags()) def gen_objectid(): return lambda: ObjectId() def gen_dbref(): collection = gen_unicode(gen_range(0, 20)) return lambda: DBRef(collection(), gen_mongo_value(1, True)()) def gen_mongo_value(depth, ref): bintype = Binary if PY3: # If we used Binary in python3 tests would fail since we # decode BSON binary subtype 0 to bytes. Testing this with # bytes in python3 makes a lot more sense. bintype = bytes choices = [gen_unicode(gen_range(0, 50)), gen_printable_string(gen_range(0, 50)), my_map(gen_string(gen_range(0, 1000)), bintype), gen_int(), gen_float(), gen_boolean(), gen_datetime(), gen_objectid(), lift(None)] if ref: choices.append(gen_dbref()) if depth > 0: choices.append(gen_mongo_list(depth, ref)) choices.append(gen_mongo_dict(depth, ref)) return choose(choices) def gen_mongo_list(depth, ref): return gen_list(gen_mongo_value(depth - 1, ref), gen_range(0, 10)) def gen_mongo_dict(depth, ref=True): return my_map(gen_dict(gen_unicode(gen_range(0, 20)), gen_mongo_value(depth - 1, ref), gen_range(0, 10)), SON) def simplify(case): # TODO this is a hack if isinstance(case, SON) and "$ref" not in case: simplified = SON(case) # make a copy! if random.choice([True, False]): # delete simplified_keys = list(simplified) if not len(simplified_keys): return (False, case) simplified.pop(random.choice(simplified_keys)) return (True, simplified) else: # simplify a value simplified_items = list(iteritems(simplified)) if not len(simplified_items): return (False, case) (key, value) = random.choice(simplified_items) (success, value) = simplify(value) simplified[key] = value return (success, success and simplified or case) if isinstance(case, list): simplified = list(case) if random.choice([True, False]): # delete if not len(simplified): return (False, case) simplified.pop(random.randrange(len(simplified))) return (True, simplified) else: # simplify an item if not len(simplified): return (False, case) index = random.randrange(len(simplified)) (success, value) = simplify(simplified[index]) simplified[index] = value return (success, success and simplified or case) return (False, case) def reduce(case, predicate, reductions=0): for _ in range(reduction_attempts): (reduced, simplified) = simplify(case) if reduced and not predicate(simplified): return reduce(simplified, predicate, reductions + 1) return (reductions, case) def isnt(predicate): return lambda x: not predicate(x) def check(predicate, generator): counter_examples = [] for _ in range(gen_target): case = generator() try: if not predicate(case): reduction = reduce(case, predicate) counter_examples.append("after %s reductions: %r" % reduction) except: counter_examples.append("%r : %s" % (case, traceback.format_exc())) return counter_examples def check_unittest(test, predicate, generator): counter_examples = check(predicate, generator) if counter_examples: failures = len(counter_examples) message = "\n".join([" -> %s" % f for f in counter_examples[:examples]]) message = ("found %d counter examples, displaying first %d:\n%s" % (failures, min(failures, examples), message)) test.fail(message) pymongo-3.6.1/test/test_json_util.py0000644000076600000240000004204113245621354020037 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test some utilities for working with JSON and PyMongo.""" import datetime import json import re import sys import uuid sys.path[0:0] = [""] from pymongo.errors import ConfigurationError from bson import json_util, EPOCH_AWARE, EPOCH_NAIVE, SON from bson.json_util import (DatetimeRepresentation, STRICT_JSON_OPTIONS, _HAS_OBJECT_PAIRS_HOOK) from bson.binary import (ALL_UUID_REPRESENTATIONS, Binary, MD5_SUBTYPE, USER_DEFINED_SUBTYPE, JAVA_LEGACY, CSHARP_LEGACY, STANDARD) from bson.code import Code from bson.dbref import DBRef from bson.int64 import Int64 from bson.max_key import MaxKey from bson.min_key import MinKey from bson.objectid import ObjectId from bson.regex import Regex from bson.timestamp import Timestamp from bson.tz_util import FixedOffset, utc from test import unittest, IntegrationTest PY3 = sys.version_info[0] == 3 class TestJsonUtil(unittest.TestCase): def round_tripped(self, doc, **kwargs): return json_util.loads(json_util.dumps(doc, **kwargs), **kwargs) def round_trip(self, doc, **kwargs): self.assertEqual(doc, self.round_tripped(doc, **kwargs)) def test_basic(self): self.round_trip({"hello": "world"}) def test_objectid(self): self.round_trip({"id": ObjectId()}) def test_dbref(self): self.round_trip({"ref": DBRef("foo", 5)}) self.round_trip({"ref": DBRef("foo", 5, "db")}) self.round_trip({"ref": DBRef("foo", ObjectId())}) if _HAS_OBJECT_PAIRS_HOOK: # Check order. self.assertEqual( '{"$ref": "collection", "$id": 1, "$db": "db"}', json_util.dumps(DBRef('collection', 1, 'db'))) def test_datetime(self): # only millis, not micros self.round_trip({"date": datetime.datetime(2009, 12, 9, 15, 49, 45, 191000, utc)}) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000+0000"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000000+0000"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000+00:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000000+00:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000000+00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000Z"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000000Z"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) # No explicit offset jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000000"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) # Localtime behind UTC jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000-0800"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000000-0800"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000-08:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000000-08:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000000-08"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) # Localtime ahead of UTC jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000+0100"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000000+0100"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000+01:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000000+01:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000000+01"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) dtm = datetime.datetime(1, 1, 1, 1, 1, 1, 0, utc) jsn = '{"dt": {"$date": -62135593139000}}' self.assertEqual(dtm, json_util.loads(jsn)["dt"]) jsn = '{"dt": {"$date": {"$numberLong": "-62135593139000"}}}' self.assertEqual(dtm, json_util.loads(jsn)["dt"]) # Test dumps format pre_epoch = {"dt": datetime.datetime(1, 1, 1, 1, 1, 1, 10000, utc)} post_epoch = {"dt": datetime.datetime(1972, 1, 1, 1, 1, 1, 10000, utc)} self.assertEqual( '{"dt": {"$date": -62135593138990}}', json_util.dumps(pre_epoch)) self.assertEqual( '{"dt": {"$date": 63075661010}}', json_util.dumps(post_epoch)) self.assertEqual( '{"dt": {"$date": {"$numberLong": "-62135593138990"}}}', json_util.dumps(pre_epoch, json_options=STRICT_JSON_OPTIONS)) self.assertEqual( '{"dt": {"$date": "1972-01-01T01:01:01.010Z"}}', json_util.dumps(post_epoch, json_options=STRICT_JSON_OPTIONS)) number_long_options = json_util.JSONOptions( datetime_representation=DatetimeRepresentation.NUMBERLONG) self.assertEqual( '{"dt": {"$date": {"$numberLong": "63075661010"}}}', json_util.dumps(post_epoch, json_options=number_long_options)) self.assertEqual( '{"dt": {"$date": {"$numberLong": "-62135593138990"}}}', json_util.dumps(pre_epoch, json_options=number_long_options)) # ISO8601 mode assumes naive datetimes are UTC pre_epoch_naive = {"dt": datetime.datetime(1, 1, 1, 1, 1, 1, 10000)} post_epoch_naive = { "dt": datetime.datetime(1972, 1, 1, 1, 1, 1, 10000)} self.assertEqual( '{"dt": {"$date": {"$numberLong": "-62135593138990"}}}', json_util.dumps(pre_epoch_naive, json_options=STRICT_JSON_OPTIONS)) self.assertEqual( '{"dt": {"$date": "1972-01-01T01:01:01.010Z"}}', json_util.dumps(post_epoch_naive, json_options=STRICT_JSON_OPTIONS)) # Test tz_aware and tzinfo options self.assertEqual( datetime.datetime(1972, 1, 1, 1, 1, 1, 10000, utc), json_util.loads( '{"dt": {"$date": "1972-01-01T01:01:01.010+0000"}}')["dt"]) self.assertEqual( datetime.datetime(1972, 1, 1, 1, 1, 1, 10000, utc), json_util.loads( '{"dt": {"$date": "1972-01-01T01:01:01.010+0000"}}', json_options=json_util.JSONOptions(tz_aware=True, tzinfo=utc))["dt"]) self.assertEqual( datetime.datetime(1972, 1, 1, 1, 1, 1, 10000), json_util.loads( '{"dt": {"$date": "1972-01-01T01:01:01.010+0000"}}', json_options=json_util.JSONOptions(tz_aware=False))["dt"]) self.round_trip(pre_epoch_naive, json_options=json_util.JSONOptions( tz_aware=False)) # Test a non-utc timezone pacific = FixedOffset(-8 * 60, 'US/Pacific') aware_datetime = {"dt": datetime.datetime(2002, 10, 27, 6, 0, 0, 10000, pacific)} self.assertEqual( '{"dt": {"$date": "2002-10-27T06:00:00.010-0800"}}', json_util.dumps(aware_datetime, json_options=STRICT_JSON_OPTIONS)) self.round_trip(aware_datetime, json_options=json_util.JSONOptions( tz_aware=True, tzinfo=pacific)) self.round_trip(aware_datetime, json_options=json_util.JSONOptions( datetime_representation=DatetimeRepresentation.ISO8601, tz_aware=True, tzinfo=pacific)) def test_regex_object_hook(self): # Extended JSON format regular expression. pat = 'a*b' json_re = '{"$regex": "%s", "$options": "u"}' % pat loaded = json_util.object_hook(json.loads(json_re)) self.assertTrue(isinstance(loaded, Regex)) self.assertEqual(pat, loaded.pattern) self.assertEqual(re.U, loaded.flags) def test_regex(self): for regex_instance in ( re.compile("a*b", re.IGNORECASE), Regex("a*b", re.IGNORECASE)): res = self.round_tripped({"r": regex_instance})["r"] self.assertEqual("a*b", res.pattern) res = self.round_tripped({"r": Regex("a*b", re.IGNORECASE)})["r"] self.assertEqual("a*b", res.pattern) self.assertEqual(re.IGNORECASE, res.flags) unicode_options = re.I|re.M|re.S|re.U|re.X regex = re.compile("a*b", unicode_options) res = self.round_tripped({"r": regex})["r"] self.assertEqual(unicode_options, res.flags) # Some tools may not add $options if no flags are set. res = json_util.loads('{"r": {"$regex": "a*b"}}')['r'] self.assertEqual(0, res.flags) self.assertEqual( Regex('.*', 'ilm'), json_util.loads( '{"r": {"$regex": ".*", "$options": "ilm"}}')['r']) if _HAS_OBJECT_PAIRS_HOOK: # Check order. self.assertEqual( '{"$regex": ".*", "$options": "mx"}', json_util.dumps(Regex('.*', re.M | re.X))) self.assertEqual( '{"$regex": ".*", "$options": "mx"}', json_util.dumps(re.compile(b'.*', re.M | re.X))) def test_minkey(self): self.round_trip({"m": MinKey()}) def test_maxkey(self): self.round_trip({"m": MaxKey()}) def test_timestamp(self): dct = {"ts": Timestamp(4, 13)} res = json_util.dumps(dct, default=json_util.default) rtdct = json_util.loads(res) self.assertEqual(dct, rtdct) if _HAS_OBJECT_PAIRS_HOOK: self.assertEqual('{"ts": {"$timestamp": {"t": 4, "i": 13}}}', res) def test_uuid(self): doc = {'uuid': uuid.UUID('f47ac10b-58cc-4372-a567-0e02b2c3d479')} self.round_trip(doc) self.assertEqual( '{"uuid": {"$uuid": "f47ac10b58cc4372a5670e02b2c3d479"}}', json_util.dumps(doc)) if _HAS_OBJECT_PAIRS_HOOK: self.assertEqual( '{"uuid": ' '{"$binary": "9HrBC1jMQ3KlZw4CssPUeQ==", "$type": "03"}}', json_util.dumps( doc, json_options=json_util.STRICT_JSON_OPTIONS)) self.assertEqual( '{"uuid": ' '{"$binary": "9HrBC1jMQ3KlZw4CssPUeQ==", "$type": "04"}}', json_util.dumps( doc, json_options=json_util.JSONOptions( strict_uuid=True, uuid_representation=STANDARD))) self.assertEqual( doc, json_util.loads( '{"uuid": ' '{"$binary": "9HrBC1jMQ3KlZw4CssPUeQ==", "$type": "03"}}')) for uuid_representation in ALL_UUID_REPRESENTATIONS: options = json_util.JSONOptions( strict_uuid=True, uuid_representation=uuid_representation) self.round_trip(doc, json_options=options) # Ignore UUID representation when decoding BSON binary subtype 4. if _HAS_OBJECT_PAIRS_HOOK: self.assertEqual(doc, json_util.loads( '{"uuid": ' '{"$binary": "9HrBC1jMQ3KlZw4CssPUeQ==", "$type": "04"}}', json_options=options)) def test_binary(self): if PY3: bin_type_dict = {"bin": b"\x00\x01\x02\x03\x04"} else: bin_type_dict = {"bin": Binary(b"\x00\x01\x02\x03\x04")} md5_type_dict = { "md5": Binary(b' n7\x18\xaf\t/\xd1\xd1/\x80\xca\xe7q\xcc\xac', MD5_SUBTYPE)} custom_type_dict = {"custom": Binary(b"hello", USER_DEFINED_SUBTYPE)} self.round_trip(bin_type_dict) self.round_trip(md5_type_dict) self.round_trip(custom_type_dict) # Binary with subtype 0 is decoded into bytes in Python 3. bin = json_util.loads( '{"bin": {"$binary": "AAECAwQ=", "$type": "00"}}')['bin'] if PY3: self.assertEqual(type(bin), bytes) else: self.assertEqual(type(bin), Binary) # PYTHON-443 ensure old type formats are supported json_bin_dump = json_util.dumps(bin_type_dict) self.assertTrue('"$type": "00"' in json_bin_dump) self.assertEqual(bin_type_dict, json_util.loads('{"bin": {"$type": 0, "$binary": "AAECAwQ="}}')) if _HAS_OBJECT_PAIRS_HOOK: json_bin_dump = json_util.dumps(md5_type_dict) # Check order. self.assertEqual( '{"md5": {"$binary": "IG43GK8JL9HRL4DK53HMrA==",' + ' "$type": "05"}}', json_bin_dump) self.assertEqual(md5_type_dict, json_util.loads('{"md5": {"$type": 5, "$binary":' ' "IG43GK8JL9HRL4DK53HMrA=="}}')) json_bin_dump = json_util.dumps(custom_type_dict) self.assertTrue('"$type": "80"' in json_bin_dump) self.assertEqual(custom_type_dict, json_util.loads('{"custom": {"$type": 128, "$binary":' ' "aGVsbG8="}}')) # Handle mongoexport where subtype >= 128 self.assertEqual(128, json_util.loads('{"custom": {"$type": "ffffff80", "$binary":' ' "aGVsbG8="}}')['custom'].subtype) self.assertEqual(255, json_util.loads('{"custom": {"$type": "ffffffff", "$binary":' ' "aGVsbG8="}}')['custom'].subtype) def test_code(self): self.round_trip({"code": Code("function x() { return 1; }")}) code = Code("return z", z=2) res = json_util.dumps(code) self.assertEqual(code, json_util.loads(res)) if _HAS_OBJECT_PAIRS_HOOK: # Check order. self.assertEqual('{"$code": "return z", "$scope": {"z": 2}}', res) no_scope = Code('function() {}') self.assertEqual( '{"$code": "function() {}"}', json_util.dumps(no_scope)) def test_undefined(self): jsn = '{"name": {"$undefined": true}}' self.assertIsNone(json_util.loads(jsn)['name']) def test_numberlong(self): jsn = '{"weight": {"$numberLong": "65535"}}' self.assertEqual(json_util.loads(jsn)['weight'], Int64(65535)) self.assertEqual(json_util.dumps({"weight": Int64(65535)}), '{"weight": 65535}') json_options = json_util.JSONOptions(strict_number_long=True) self.assertEqual(json_util.dumps({"weight": Int64(65535)}, json_options=json_options), jsn) def test_loads_document_class(self): # document_class dict should always work self.assertEqual({"foo": "bar"}, json_util.loads( '{"foo": "bar"}', json_options=json_util.JSONOptions(document_class=dict))) if not _HAS_OBJECT_PAIRS_HOOK: self.assertRaises( ConfigurationError, json_util.JSONOptions, document_class=SON) else: self.assertEqual(SON([("foo", "bar"), ("b", 1)]), json_util.loads( '{"foo": "bar", "b": 1}', json_options=json_util.JSONOptions(document_class=SON))) class TestJsonUtilRoundtrip(IntegrationTest): def test_cursor(self): db = self.db db.drop_collection("test") docs = [ {'foo': [1, 2]}, {'bar': {'hello': 'world'}}, {'code': Code("function x() { return 1; }")}, {'bin': Binary(b"\x00\x01\x02\x03\x04", USER_DEFINED_SUBTYPE)}, {'dbref': {'_ref': DBRef('simple', ObjectId('509b8db456c02c5ab7e63c34'))}} ] db.test.insert_many(docs) reloaded_docs = json_util.loads(json_util.dumps(db.test.find())) for doc in docs: self.assertTrue(doc in reloaded_docs) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_auth.py0000644000076600000240000004256113245621354017001 0ustar shanestaff00000000000000# Copyright 2013-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Authentication Tests.""" import os import sys import threading try: from urllib.parse import quote_plus except ImportError: # Python 2 from urllib import quote_plus sys.path[0:0] = [""] from pymongo import MongoClient from pymongo.auth import HAVE_KERBEROS, _build_credentials_tuple from pymongo.errors import OperationFailure from pymongo.read_preferences import ReadPreference from test import client_context, SkipTest, unittest, Version from test.utils import (delay, ignore_deprecations, rs_or_single_client_noauth, single_client_noauth) # YOU MUST RUN KINIT BEFORE RUNNING GSSAPI TESTS ON UNIX. GSSAPI_HOST = os.environ.get('GSSAPI_HOST') GSSAPI_PORT = int(os.environ.get('GSSAPI_PORT', '27017')) GSSAPI_PRINCIPAL = os.environ.get('GSSAPI_PRINCIPAL') GSSAPI_SERVICE_NAME = os.environ.get('GSSAPI_SERVICE_NAME', 'mongodb') GSSAPI_CANONICALIZE = os.environ.get('GSSAPI_CANONICALIZE', 'false') GSSAPI_SERVICE_REALM = os.environ.get('GSSAPI_SERVICE_REALM') GSSAPI_PASS = os.environ.get('GSSAPI_PASS') GSSAPI_DB = os.environ.get('GSSAPI_DB', 'test') SASL_HOST = os.environ.get('SASL_HOST') SASL_PORT = int(os.environ.get('SASL_PORT', '27017')) SASL_USER = os.environ.get('SASL_USER') SASL_PASS = os.environ.get('SASL_PASS') SASL_DB = os.environ.get('SASL_DB', '$external') class AutoAuthenticateThread(threading.Thread): """Used in testing threaded authentication. This does collection.find_one() with a 1-second delay to ensure it must check out and authenticate multiple sockets from the pool concurrently. :Parameters: `collection`: An auth-protected collection containing one document. """ def __init__(self, collection): super(AutoAuthenticateThread, self).__init__() self.collection = collection self.success = False def run(self): assert self.collection.find_one({'$where': delay(1)}) is not None self.success = True class TestGSSAPI(unittest.TestCase): @classmethod def setUpClass(cls): if not HAVE_KERBEROS: raise SkipTest('Kerberos module not available.') if not GSSAPI_HOST or not GSSAPI_PRINCIPAL: raise SkipTest( 'Must set GSSAPI_HOST and GSSAPI_PRINCIPAL to test GSSAPI') cls.service_realm_required = ( GSSAPI_SERVICE_REALM is not None and GSSAPI_SERVICE_REALM not in GSSAPI_PRINCIPAL) mech_properties = 'SERVICE_NAME:%s' % (GSSAPI_SERVICE_NAME,) mech_properties += ( ',CANONICALIZE_HOST_NAME:%s' % (GSSAPI_CANONICALIZE,)) if GSSAPI_SERVICE_REALM is not None: mech_properties += ',SERVICE_REALM:%s' % (GSSAPI_SERVICE_REALM,) cls.mech_properties = mech_properties def test_credentials_hashing(self): # GSSAPI credentials are properly hashed. creds0 = _build_credentials_tuple( 'GSSAPI', '', 'user', 'pass', {}) creds1 = _build_credentials_tuple( 'GSSAPI', '', 'user', 'pass', {'authmechanismproperties': {'SERVICE_NAME': 'A'}}) creds2 = _build_credentials_tuple( 'GSSAPI', '', 'user', 'pass', {'authmechanismproperties': {'SERVICE_NAME': 'A'}}) creds3 = _build_credentials_tuple( 'GSSAPI', '', 'user', 'pass', {'authmechanismproperties': {'SERVICE_NAME': 'B'}}) self.assertEqual(1, len(set([creds1, creds2]))) self.assertEqual(3, len(set([creds0, creds1, creds2, creds3]))) @ignore_deprecations def test_gssapi_simple(self): if GSSAPI_PASS is not None: uri = ('mongodb://%s:%s@%s:%d/?authMechanism=' 'GSSAPI' % (quote_plus(GSSAPI_PRINCIPAL), GSSAPI_PASS, GSSAPI_HOST, GSSAPI_PORT)) else: uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI' % (quote_plus(GSSAPI_PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) if not self.service_realm_required: # Without authMechanismProperties. client = MongoClient(GSSAPI_HOST, GSSAPI_PORT, username=GSSAPI_PRINCIPAL, password=GSSAPI_PASS, authMechanism='GSSAPI') client[GSSAPI_DB].collection.find_one() # Log in using URI, without authMechanismProperties. client = MongoClient(uri) client[GSSAPI_DB].collection.find_one() # Authenticate with authMechanismProperties. client = MongoClient(GSSAPI_HOST, GSSAPI_PORT, username=GSSAPI_PRINCIPAL, password=GSSAPI_PASS, authMechanism='GSSAPI', authMechanismProperties=self.mech_properties) client[GSSAPI_DB].collection.find_one() # Log in using URI, with authMechanismProperties. mech_uri = uri + '&authMechanismProperties=%s' % (self.mech_properties,) client = MongoClient(mech_uri) client[GSSAPI_DB].collection.find_one() set_name = client.admin.command('ismaster').get('setName') if set_name: if not self.service_realm_required: # Without authMechanismProperties client = MongoClient(GSSAPI_HOST, GSSAPI_PORT, username=GSSAPI_PRINCIPAL, password=GSSAPI_PASS, authMechanism='GSSAPI', replicaSet=set_name) client[GSSAPI_DB].collection_names() uri = uri + '&replicaSet=%s' % (str(set_name),) client = MongoClient(uri) client[GSSAPI_DB].collection_names() # With authMechanismProperties client = MongoClient(GSSAPI_HOST, GSSAPI_PORT, username=GSSAPI_PRINCIPAL, password=GSSAPI_PASS, authMechanism='GSSAPI', authMechanismProperties=self.mech_properties, replicaSet=set_name) client[GSSAPI_DB].collection_names() mech_uri = mech_uri + '&replicaSet=%s' % (str(set_name),) client = MongoClient(mech_uri) client[GSSAPI_DB].collection_names() @ignore_deprecations def test_gssapi_threaded(self): client = MongoClient(GSSAPI_HOST, GSSAPI_PORT, username=GSSAPI_PRINCIPAL, password=GSSAPI_PASS, authMechanism='GSSAPI', authMechanismProperties=self.mech_properties) # Authentication succeeded? client.server_info() db = client[GSSAPI_DB] # Need one document in the collection. AutoAuthenticateThread does # collection.find_one with a 1-second delay, forcing it to check out # multiple sockets from the pool concurrently, proving that # auto-authentication works with GSSAPI. collection = db.test if collection.count() == 0: try: collection.drop() collection.insert_one({'_id': 1}) except OperationFailure: raise SkipTest("User must be able to write.") threads = [] for _ in range(4): threads.append(AutoAuthenticateThread(collection)) for thread in threads: thread.start() for thread in threads: thread.join() self.assertTrue(thread.success) set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoClient(GSSAPI_HOST, GSSAPI_PORT, username=GSSAPI_PRINCIPAL, password=GSSAPI_PASS, authMechanism='GSSAPI', authMechanismProperties=self.mech_properties, replicaSet=set_name) # Succeeded? client.server_info() threads = [] for _ in range(4): threads.append(AutoAuthenticateThread(collection)) for thread in threads: thread.start() for thread in threads: thread.join() self.assertTrue(thread.success) class TestSASLPlain(unittest.TestCase): @classmethod def setUpClass(cls): if not SASL_HOST or not SASL_USER or not SASL_PASS: raise SkipTest('Must set SASL_HOST, ' 'SASL_USER, and SASL_PASS to test SASL') def test_sasl_plain(self): client = MongoClient(SASL_HOST, SASL_PORT, username=SASL_USER, password=SASL_PASS, authSource=SASL_DB, authMechanism='PLAIN') client.ldap.test.find_one() uri = ('mongodb://%s:%s@%s:%d/?authMechanism=PLAIN;' 'authSource=%s' % (quote_plus(SASL_USER), quote_plus(SASL_PASS), SASL_HOST, SASL_PORT, SASL_DB)) client = MongoClient(uri) client.ldap.test.find_one() set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoClient(SASL_HOST, SASL_PORT, replicaSet=set_name, username=SASL_USER, password=SASL_PASS, authSource=SASL_DB, authMechanism='PLAIN') client.ldap.test.find_one() uri = ('mongodb://%s:%s@%s:%d/?authMechanism=PLAIN;' 'authSource=%s;replicaSet=%s' % (quote_plus(SASL_USER), quote_plus(SASL_PASS), SASL_HOST, SASL_PORT, SASL_DB, str(set_name))) client = MongoClient(uri) client.ldap.test.find_one() def test_sasl_plain_bad_credentials(self): with ignore_deprecations(): client = MongoClient(SASL_HOST, SASL_PORT) # Bad username self.assertRaises(OperationFailure, client.ldap.authenticate, 'not-user', SASL_PASS, SASL_DB, 'PLAIN') self.assertRaises(OperationFailure, client.ldap.test.find_one) self.assertRaises(OperationFailure, client.ldap.test.insert_one, {"failed": True}) # Bad password self.assertRaises(OperationFailure, client.ldap.authenticate, SASL_USER, 'not-pwd', SASL_DB, 'PLAIN') self.assertRaises(OperationFailure, client.ldap.test.find_one) self.assertRaises(OperationFailure, client.ldap.test.insert_one, {"failed": True}) def auth_string(user, password): uri = ('mongodb://%s:%s@%s:%d/?authMechanism=PLAIN;' 'authSource=%s' % (quote_plus(user), quote_plus(password), SASL_HOST, SASL_PORT, SASL_DB)) return uri bad_user = MongoClient(auth_string('not-user', SASL_PASS)) bad_pwd = MongoClient(auth_string(SASL_USER, 'not-pwd')) # OperationFailure raised upon connecting. self.assertRaises(OperationFailure, bad_user.admin.command, 'ismaster') self.assertRaises(OperationFailure, bad_pwd.admin.command, 'ismaster') class TestSCRAMSHA1(unittest.TestCase): @client_context.require_auth @client_context.require_version_min(2, 7, 2) def setUp(self): # Before 2.7.7, SCRAM-SHA-1 had to be enabled from the command line. if client_context.version < Version(2, 7, 7): cmd_line = client_context.cmd_line if 'SCRAM-SHA-1' not in cmd_line.get( 'parsed', {}).get('setParameter', {}).get('authenticationMechanisms', ''): raise SkipTest('SCRAM-SHA-1 mechanism not enabled') client_context.create_user( 'pymongo_test', 'user', 'pass', roles=['userAdmin', 'readWrite']) def tearDown(self): client_context.drop_user('pymongo_test', 'user') def test_scram_sha1(self): host, port = client_context.host, client_context.port with ignore_deprecations(): client = rs_or_single_client_noauth() self.assertTrue(client.pymongo_test.authenticate( 'user', 'pass', mechanism='SCRAM-SHA-1')) client.pymongo_test.command('dbstats') client = rs_or_single_client_noauth( 'mongodb://user:pass@%s:%d/pymongo_test?authMechanism=SCRAM-SHA-1' % (host, port)) client.pymongo_test.command('dbstats') if client_context.is_rs: uri = ('mongodb://user:pass' '@%s:%d/pymongo_test?authMechanism=SCRAM-SHA-1' '&replicaSet=%s' % (host, port, client_context.replica_set_name)) client = single_client_noauth(uri) client.pymongo_test.command('dbstats') db = client.get_database( 'pymongo_test', read_preference=ReadPreference.SECONDARY) db.command('dbstats') class TestAuthURIOptions(unittest.TestCase): @client_context.require_auth def setUp(self): client_context.create_user('admin', 'admin', 'pass') client_context.create_user( 'pymongo_test', 'user', 'pass', ['userAdmin', 'readWrite']) self.client = rs_or_single_client_noauth( username='admin', password='pass') def tearDown(self): client_context.drop_user('pymongo_test', 'user') client_context.drop_user('admin', 'admin') def test_uri_options(self): # Test default to admin host, port = client_context.host, client_context.port client = rs_or_single_client_noauth( 'mongodb://admin:pass@%s:%d' % (host, port)) self.assertTrue(client.admin.command('dbstats')) if client_context.is_rs: uri = ('mongodb://admin:pass@%s:%d/?replicaSet=%s' % ( host, port, client_context.replica_set_name)) client = single_client_noauth(uri) self.assertTrue(client.admin.command('dbstats')) db = client.get_database( 'admin', read_preference=ReadPreference.SECONDARY) self.assertTrue(db.command('dbstats')) # Test explicit database uri = 'mongodb://user:pass@%s:%d/pymongo_test' % (host, port) client = rs_or_single_client_noauth(uri) self.assertRaises(OperationFailure, client.admin.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) if client_context.is_rs: uri = ('mongodb://user:pass@%s:%d/pymongo_test?replicaSet=%s' % ( host, port, client_context.replica_set_name)) client = single_client_noauth(uri) self.assertRaises(OperationFailure, client.admin.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) db = client.get_database( 'pymongo_test', read_preference=ReadPreference.SECONDARY) self.assertTrue(db.command('dbstats')) # Test authSource uri = ('mongodb://user:pass@%s:%d' '/pymongo_test2?authSource=pymongo_test' % (host, port)) client = rs_or_single_client_noauth(uri) self.assertRaises(OperationFailure, client.pymongo_test2.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) if client_context.is_rs: uri = ('mongodb://user:pass@%s:%d/pymongo_test2?replicaSet=' '%s;authSource=pymongo_test' % ( host, port, client_context.replica_set_name)) client = single_client_noauth(uri) self.assertRaises(OperationFailure, client.pymongo_test2.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) db = client.get_database( 'pymongo_test', read_preference=ReadPreference.SECONDARY) self.assertTrue(db.command('dbstats')) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_heartbeat_monitoring.py0000644000076600000240000001125713245621354022242 0ustar shanestaff00000000000000# Copyright 2016-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the monitoring of the server heartbeats.""" import sys import threading sys.path[0:0] = [""] from pymongo import monitoring from pymongo.errors import ConnectionFailure from pymongo.ismaster import IsMaster from pymongo.monitor import Monitor from pymongo.pool import PoolOptions from test import unittest, client_knobs from test.utils import HeartbeatEventListener, single_client, wait_until class MockSocketInfo(object): def close(self): pass def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass class MockPool(object): def __init__(self, *args, **kwargs): self.pool_id = 0 self._lock = threading.Lock() self.opts = PoolOptions() def get_socket(self, all_credentials): return MockSocketInfo() def return_socket(self, _): pass def reset(self): with self._lock: self.pool_id += 1 def remove_stale_sockets(self): pass class TestHeartbeatMonitoring(unittest.TestCase): @classmethod def setUpClass(cls): cls.saved_listeners = monitoring._LISTENERS monitoring._LISTENERS = monitoring._Listeners([], [], [], []) @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners def create_mock_monitor(self, responses, uri, expected_results): listener = HeartbeatEventListener() with client_knobs(heartbeat_frequency=0.1, min_heartbeat_interval=0.1, events_queue_frequency=0.1): class MockMonitor(Monitor): def _check_with_socket(self, *args, **kwargs): if isinstance(responses[1], Exception): raise responses[1] return IsMaster(responses[1]), 99 m = single_client( h=uri, event_listeners=(listener,), _monitor_class=MockMonitor, _pool_class=MockPool) expected_len = len(expected_results) # Wait for *at least* expected_len number of results. The # monitor thread may run multiple times during the execution # of this test. wait_until( lambda: len(listener.results) >= expected_len, "publish all events") try: # zip gives us len(expected_results) pairs. for expected, actual in zip(expected_results, listener.results): self.assertEqual(expected, actual.__class__.__name__) self.assertEqual(actual.connection_id, responses[0]) if expected != 'ServerHeartbeatStartedEvent': if isinstance(actual.reply, IsMaster): self.assertEqual(actual.duration, 99) self.assertEqual(actual.reply._doc, responses[1]) else: self.assertEqual(actual.reply, responses[1]) finally: m.close() def test_standalone(self): responses = (('a', 27017), { "ismaster": True, "maxWireVersion": 4, "minWireVersion": 0, "ok": 1 }) uri = "mongodb://a:27017" expected_results = ['ServerHeartbeatStartedEvent', 'ServerHeartbeatSucceededEvent'] self.create_mock_monitor(responses, uri, expected_results) def test_standalone_error(self): responses = (('a', 27017), ConnectionFailure("SPECIAL MESSAGE")) uri = "mongodb://a:27017" # _check_with_socket failing results in a second attempt. expected_results = ['ServerHeartbeatStartedEvent', 'ServerHeartbeatFailedEvent', 'ServerHeartbeatStartedEvent', 'ServerHeartbeatFailedEvent'] self.create_mock_monitor(responses, uri, expected_results) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/high_availability/0000755000076600000240000000000013246104133020060 5ustar shanestaff00000000000000pymongo-3.6.1/test/high_availability/test_ha.py0000644000076600000240000011347313245621354022102 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test replica set operations and failures.""" # These test methods exuberantly violate the "one assert per test" rule, because # each method requires running setUp, which takes about 30 seconds to bring up # a replica set. Thus each method asserts everything we want to assert for a # given replica-set configuration. import time import ha_tools from pymongo import common from pymongo.common import partition_node from pymongo.errors import (AutoReconnect, OperationFailure, ConnectionFailure, InvalidOperation, WTimeoutError) from pymongo.mongo_client import MongoClient from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.write_concern import WriteConcern from test import unittest, utils, client_knobs from test.utils import one, wait_until, connected # To make the code terser, copy modes into module scope PRIMARY = ReadPreference.PRIMARY PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED SECONDARY = ReadPreference.SECONDARY SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED NEAREST = ReadPreference.NEAREST def partition_nodes(nodes): """Translate from ['host:port', ...] to [(host, port), ...]""" return [partition_node(node) for node in nodes] class HATestCase(unittest.TestCase): """A test case for connections to replica sets or mongos.""" # Override default 10-second interval for faster testing... heartbeat_frequency = 0.5 # ... or disable it by setting "enable_heartbeat" to False. enable_heartbeat = True # Override this to speed up connection-failure tests. server_selection_timeout = common.SERVER_SELECTION_TIMEOUT def setUp(self): if self.enable_heartbeat: heartbeat_frequency = self.heartbeat_frequency else: # Disable periodic monitoring. heartbeat_frequency = 1e6 self.knobs = client_knobs(heartbeat_frequency=heartbeat_frequency) self.knobs.enable() def tearDown(self): ha_tools.kill_all_members() ha_tools.nodes.clear() ha_tools.routers.clear() time.sleep(1) # Let members really die. self.knobs.disable() class TestDirectConnection(HATestCase): def setUp(self): super(TestDirectConnection, self).setUp() members = [{}, {}, {'arbiterOnly': True}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_secondary_connection(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: len(self.c.secondaries), "discover secondary") # Wait for replication... w = len(self.c.secondaries) + 1 db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # MongoClient succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, ]: client = MongoClient( primary_host, primary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: primary_host == client.host, "connect to primary") self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(client.pymongo_test.test.find_one()) client = MongoClient( secondary_host, secondary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: secondary_host == client.host, "connect to secondary") self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(client.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, client.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MongoClient # should do the same for unacknowledged writes. try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % (client,)) # Test direct connection to an arbiter client = MongoClient( arbiter_host, arbiter_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: arbiter_host == client.host, "connect to arbiter") self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into arbiter client %s should' 'have raised exception' % (client,)) class TestPassiveAndHidden(HATestCase): def setUp(self): super(TestPassiveAndHidden, self).setUp() members = [{}, {'priority': 0}, {'arbiterOnly': True}, {'priority': 0, 'hidden': True}, {'priority': 0, 'slaveDelay': 5} ] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_passive_and_hidden(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) passives = ha_tools.get_passives() passives = partition_nodes(passives) self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: utils.assertReadFromAll(self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) time.sleep(2 * self.heartbeat_frequency) utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED) class TestMonitorRemovesRecoveringMember(HATestCase): # Members in STARTUP2 or RECOVERING states are shown in the primary's # isMaster response, but aren't secondaries and shouldn't be read from. # Verify that if a secondary goes into RECOVERING mode, the Monitor removes # it from the set of readers. def setUp(self): super(TestMonitorRemovesRecoveringMember, self).setUp() members = [{}, {'priority': 0}, {'priority': 0}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_monitor_removes_recovering_member(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = partition_nodes(secondaries) utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) time.sleep(2 * self.heartbeat_frequency) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member utils.assertReadFrom(self, self.c, partition_node(secondary), mode) class TestTriggeredRefresh(HATestCase): # Verify that if a secondary goes into RECOVERING mode or if the primary # changes, the next exception triggers an immediate refresh. enable_heartbeat = False def setUp(self): super(TestTriggeredRefresh, self).setUp() members = [{}, {}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_recovering_member_triggers_refresh(self): # To test that find_one() and count() trigger immediate refreshes, # we'll create a separate client for each self.c_find_one, self.c_count = [ MongoClient( self.seed, replicaSet=self.name, read_preference=SECONDARY, serverSelectionTimeoutMS=self.server_selection_timeout) for _ in xrange(2)] # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] # Pre-condition: just make sure they all connected OK for c in self.c_find_one, self.c_count: wait_until( lambda: c.primary == partition_node(primary), 'connect to the primary') wait_until( lambda: one(c.secondaries) == partition_node(secondary), 'connect to the secondary') ha_tools.set_maintenance(secondary, True) # Trigger a refresh in various ways self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one) self.assertRaises(AutoReconnect, self.c_count.test.test.count) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled time.sleep(1) self.assertFalse(self.c_find_one.secondaries) self.assertEqual(partition_node(primary), self.c_find_one.primary) self.assertFalse(self.c_count.secondaries) self.assertEqual(partition_node(primary), self.c_count.primary) def test_stepdown_triggers_refresh(self): c_find_one = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) c_count = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) # We've started the primary and one secondary wait_until(lambda: len(c_find_one.secondaries), "discover secondary") wait_until(lambda: len(c_count.secondaries), "discover secondary") ha_tools.stepdown_primary() # Trigger a refresh, both with a cursor and a command. self.assertRaises(AutoReconnect, c_find_one.test.test.find_one) self.assertRaises(AutoReconnect, c_count.test.command, 'count') # Both clients detect the stepdown *AND* re-check the server # immediately, they don't just mark it Unknown. Wait for the # immediate refresh to complete - we're not waiting for the # periodic refresh, which has been disabled wait_until(lambda: len(c_find_one.secondaries) == 2, "detect two secondaries") wait_until(lambda: len(c_count.secondaries) == 2, "detect two secondaries") class TestHealthMonitor(HATestCase): def setUp(self): super(TestHealthMonitor, self).setUp() res = ha_tools.start_replica_set([{}, {}, {}]) self.seed, self.name = res def test_primary_failure(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") old_primary = c.primary old_secondaries = c.secondaries killed = ha_tools.kill_primary() self.assertTrue(bool(len(killed))) wait_until(lambda: c.primary and c.primary != old_primary, "discover new primary", timeout=30) wait_until(lambda: c.secondaries != old_secondaries, "discover new secondaries", timeout=30) def test_secondary_failure(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") primary = c.primary old_secondaries = c.secondaries killed = ha_tools.kill_secondary() time.sleep(2 * self.heartbeat_frequency) self.assertTrue(bool(len(killed))) self.assertEqual(primary, c.primary) wait_until(lambda: c.secondaries != old_secondaries, "discover new secondaries", timeout=30) old_secondaries = c.secondaries ha_tools.restart_members([killed]) self.assertEqual(primary, c.primary) wait_until(lambda: c.secondaries != old_secondaries, "discover new secondaries", timeout=30) def test_primary_stepdown(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") ha_tools.stepdown_primary() # Wait for new primary. wait_until(lambda: (ha_tools.get_primary() and c.primary == partition_node(ha_tools.get_primary())), "discover new primary", timeout=30) wait_until(lambda: len(c.secondaries) == 2, "discover new secondaries", timeout=30) class TestWritesWithFailover(HATestCase): enable_heartbeat = False def setUp(self): super(TestWritesWithFailover, self).setUp() res = ha_tools.start_replica_set([{}, {}, {}]) self.seed, self.name = res def test_writes_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") primary = c.primary w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) self.assertEqual('bar', db.test.find_one()['foo']) killed = ha_tools.kill_primary(9) self.assertTrue(bool(len(killed))) # Wait past pool's check interval, so it throws an error from # get_socket(). time.sleep(1) # Verify that we only raise AutoReconnect, not some other error, # while we wait for new primary. for _ in xrange(10000): try: db.test.insert_one({'bar': 'baz'}) # No error, found primary. break except AutoReconnect: time.sleep(.01) else: self.fail("Couldn't connect to new primary") # Found new primary. self.assertTrue(c.primary) self.assertTrue(primary != c.primary) self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar']) class TestReadWithFailover(HATestCase): def setUp(self): super(TestReadWithFailover, self).setUp() res = ha_tools.start_replica_set([{}, {}, {}]) self.seed, self.name = res def test_read_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def iter_cursor(cursor): for _ in cursor: pass return True w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) # Force replication db.test.insert_many([{'foo': i} for i in xrange(10)]) self.assertEqual(10, db.test.count()) db.read_preference = SECONDARY_PREFERRED cursor = db.test.find().batch_size(5) next(cursor) self.assertEqual(5, cursor._Cursor__retrieved) self.assertTrue(cursor.address in c.secondaries) ha_tools.kill_primary() # Primary failure shouldn't interrupt the cursor self.assertTrue(iter_cursor(cursor)) self.assertEqual(10, cursor._Cursor__retrieved) class TestReadPreference(HATestCase): # Speed up assertReadFrom() when no server is suitable. server_selection_timeout = 0.001 def setUp(self): super(TestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) self.w = len(self.c.secondaries) + 1 self.db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=self.w)) self.db.test.delete_many({}) self.db.test.insert_many([{'foo': i} for i in xrange(10)]) self.clear_ping_times() def set_ping_time(self, host, ping_time_seconds): ServerDescription._host_to_round_trip_time[host] = ping_time_seconds def clear_ping_times(self): ServerDescription._host_to_round_trip_time.clear() def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone time.sleep(2 * self.heartbeat_frequency) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) ha_tools.wait_for_primary() ha_tools.kill_members([unpartition_node(secondary)], 2) time.sleep(5) ha_tools.wait_for_primary() time.sleep(2 * self.heartbeat_frequency) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times() class TestReplicaSetAuth(HATestCase): def setUp(self): super(TestReplicaSetAuth, self).setUp() members = [ {}, {'priority': 0}, {'priority': 0}, ] res = ha_tools.start_replica_set(members, auth=True) self.c = MongoClient( res[0], replicaSet=res[1], serverSelectionTimeoutMS=self.server_selection_timeout) # Add an admin user to enable auth self.c.admin.add_user('admin', 'adminpass') self.c.admin.authenticate('admin', 'adminpass') self.db = self.c.pymongo_ha_auth self.db.add_user('user', 'userpass') self.c.admin.logout() def test_auth_during_failover(self): self.assertTrue(self.db.authenticate('user', 'userpass')) db = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=3, wtimeout=3000)) self.assertTrue(db.foo.insert_one({'foo': 'bar'})) self.db.logout() self.assertRaises(OperationFailure, self.db.foo.find_one) primary = self.c.primary ha_tools.kill_members(['%s:%d' % primary], 2) # Let monitor notice primary's gone time.sleep(2 * self.heartbeat_frequency) self.assertFalse(primary == self.c.primary) # Make sure we can still authenticate self.assertTrue(self.db.authenticate('user', 'userpass')) # And still query. self.db.read_preference = PRIMARY_PREFERRED self.assertEqual('bar', self.db.foo.find_one()['foo']) class TestAlive(HATestCase): def setUp(self): super(TestAlive, self).setUp() members = [{}, {}] self.seed, self.name = ha_tools.start_replica_set(members) def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = connected( MongoClient( primary, serverSelectionTimeoutMS=self.server_selection_timeout)), secondary_cx = connected( MongoClient( secondary, serverSelectionTimeoutMS=self.server_selection_timeout)) rsc = connected( MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout)) self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) class TestMongosLoadBalancing(HATestCase): def setUp(self): super(TestMongosLoadBalancing, self).setUp() seed_list = ha_tools.create_sharded_cluster() self.assertIsNotNone(seed_list) self.dbname = 'pymongo_mongos_ha' self.client = MongoClient( seed_list, serverSelectionTimeoutMS=self.server_selection_timeout) self.client.drop_database(self.dbname) def test_mongos_load_balancing(self): wait_until(lambda: len(ha_tools.routers) == len(self.client.nodes), 'discover all mongoses') # Can't access "address" when load balancing. with self.assertRaises(InvalidOperation): self.client.address coll = self.client[self.dbname].test coll.insert_one({'foo': 'bar'}) live_routers = list(ha_tools.routers) ha_tools.kill_mongos(live_routers.pop()) while live_routers: try: self.assertEqual(1, coll.count()) except ConnectionFailure: # If first attempt happened to select the dead mongos. self.assertEqual(1, coll.count()) wait_until(lambda: len(live_routers) == len(self.client.nodes), 'remove dead mongos', timeout=30) ha_tools.kill_mongos(live_routers.pop()) # Make sure the last one's really dead. time.sleep(1) # I'm alone. self.assertRaises(ConnectionFailure, coll.count) wait_until(lambda: 0 == len(self.client.nodes), 'remove dead mongos', timeout=30) ha_tools.restart_mongos(one(ha_tools.routers)) # Find new mongos self.assertEqual(1, coll.count()) class TestLastErrorDefaults(HATestCase): def setUp(self): super(TestLastErrorDefaults, self).setUp() members = [{}, {}] res = ha_tools.start_replica_set(members) self.seed, self.name = res self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) def test_get_last_error_defaults(self): replset = self.c.local.system.replset.find_one() settings = replset.get('settings', {}) # This should cause a WTimeoutError for every write command settings['getLastErrorDefaults'] = { 'w': 3, 'wtimeout': 1 } replset['settings'] = settings replset['version'] = replset.get("version", 1) + 1 self.c.admin.command("replSetReconfig", replset) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.insert_one, {'_id': 0}) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.update_one, {'_id': 0}, {"$set": {"a": 10}}) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.delete_one, {'_id': 0}) class TestShipOfTheseus(HATestCase): # If all of a replica set's members are replaced with new ones, is it still # the same replica set, or a different one? def setUp(self): super(TestShipOfTheseus, self).setUp() res = ha_tools.start_replica_set([{}, {}]) self.seed, self.name = res def test_ship_of_theseus(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=len(c.secondaries) + 1)) db.test.insert_one({}) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [] for i in range(3): new_hosts.append(ha_tools.add_member()) # RS closes all connections after reconfig. for j in xrange(30): try: if ha_tools.get_primary(): break except (ConnectionFailure, OperationFailure): pass time.sleep(1) else: self.fail("Couldn't recover from reconfig") # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break time.sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) time.sleep(5) wait_until(lambda: (ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2), "fail over", timeout=30) time.sleep(2 * self.heartbeat_frequency) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY) # Kill new members and switch back to original two members. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members([primary, secondary1]) # Wait for members to figure out they're secondaries. wait_until(lambda: len(ha_tools.get_secondaries()) == 2, "detect two secondaries", timeout=30) # Should be able to reconnect to set again. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY) class TestLastError(HATestCase): # A "not master" error from Database.error() should refresh the server. enable_heartbeat = False def setUp(self): super(TestLastError, self).setUp() res = ha_tools.start_replica_set([{}, {}]) self.seed, self.name = res def test_last_error(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: c.secondaries, "discover secondary") ha_tools.stepdown_primary() db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=0)) db.test.insert_one({}) response = db.error() self.assertTrue('err' in response and 'not master' in response['err']) wait_until(lambda: len(c.secondaries) == 2, "discover two secondaries") if __name__ == '__main__': unittest.main() pymongo-3.6.1/test/high_availability/ha_tools.py0000644000076600000240000003473613245617773022301 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for testing high availability in PyMongo.""" import os import random import shutil import signal import socket import subprocess import sys import time from stat import S_IRUSR import pymongo import pymongo.errors from pymongo.read_preferences import ReadPreference from test.utils import connected home = os.environ.get('HOME') default_dbpath = os.path.join(home, 'data', 'pymongo_high_availability') dbpath = os.environ.get('DBPATH', default_dbpath) default_logpath = os.path.join(home, 'log', 'pymongo_high_availability') logpath = os.path.expanduser(os.environ.get('LOGPATH', default_logpath)) hostname = os.environ.get('HOSTNAME', 'localhost') port = int(os.environ.get('DBPORT', 27017)) mongod = os.path.expanduser(os.environ.get('MONGOD', 'mongod')) mongos = os.path.expanduser(os.environ.get('MONGOS', 'mongos')) replica_set_name = os.environ.get('SETNAME', 'repl0') ha_tools_debug = bool(os.environ.get('HA_TOOLS_DEBUG')) nodes = {} routers = {} cur_port = port key_file = None try: from subprocess import DEVNULL # Python 3. except ImportError: DEVNULL = open(os.devnull, 'wb') def kill_members(members, sig, hosts=nodes): for member in sorted(members): try: if ha_tools_debug: print('killing %s' % (member,)) proc = hosts[member]['proc'] if 'java' in sys.platform: # _process is a wrapped java.lang.UNIXProcess. proc._process.destroy() # Not sure if cygwin makes sense here... elif sys.platform in ('win32', 'cygwin'): os.kill(proc.pid, signal.CTRL_C_EVENT) else: os.kill(proc.pid, sig) except OSError: if ha_tools_debug: print('%s already dead?' % (member,)) def kill_all_members(): kill_members(nodes.keys(), 2, nodes) kill_members(routers.keys(), 2, routers) def wait_for(proc, port_num): trys = 0 while proc.poll() is None and trys < 160: trys += 1 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: try: s.connect((hostname, port_num)) return True except (IOError, socket.error): time.sleep(0.25) finally: s.close() kill_all_members() return False def start_subprocess(cmd): """Run cmd (a list of strings) and return a Popen instance.""" return subprocess.Popen(cmd, stdout=DEVNULL, stderr=DEVNULL) def start_replica_set(members, auth=False, fresh=True): global cur_port global key_file if fresh: if os.path.exists(dbpath): try: shutil.rmtree(dbpath) except OSError: pass try: os.makedirs(dbpath) except OSError as exc: print(exc) print("\tWhile creating %s" % (dbpath,)) if auth: key_file = os.path.join(dbpath, 'key.txt') if not os.path.exists(key_file): with open(key_file, 'w') as f: f.write("my super secret system password") os.chmod(key_file, S_IRUSR) for i in range(len(members)): host = '%s:%d' % (hostname, cur_port) members[i].update({'_id': i, 'host': host}) path = os.path.join(dbpath, 'db' + str(i)) if not os.path.exists(path): os.makedirs(path) member_logpath = os.path.join(logpath, 'db' + str(i) + '.log') if not os.path.exists(os.path.dirname(member_logpath)): os.makedirs(os.path.dirname(member_logpath)) cmd = [mongod, '--dbpath', path, '--port', str(cur_port), '--replSet', replica_set_name, '--nojournal', '--oplogSize', '64', '--logappend', '--logpath', member_logpath] if auth: cmd += ['--keyFile', key_file] if ha_tools_debug: print('starting %s' % (' '.join(cmd),)) proc = start_subprocess(cmd) nodes[host] = {'proc': proc, 'cmd': cmd, 'dbpath': path} res = wait_for(proc, cur_port) cur_port += 1 if not res: return None config = {'_id': replica_set_name, 'members': members} primary = members[0]['host'] c = pymongo.MongoClient(primary) try: if ha_tools_debug: print('rs.initiate(%s)' % (config,)) c.admin.command('replSetInitiate', config) except pymongo.errors.OperationFailure as exc: # Already initialized from a previous run? if ha_tools_debug: print(exc) expected_arbiters = 0 for member in members: if member.get('arbiterOnly'): expected_arbiters += 1 expected_secondaries = len(members) - expected_arbiters - 1 # Wait a minute for replica set to come up. patience = 1 for i in range(int(patience * 60 / 2)): time.sleep(2) try: if (get_primary() and len(get_secondaries()) == expected_secondaries and len(get_arbiters()) == expected_arbiters): break except pymongo.errors.ConnectionFailure: # Keep waiting pass if ha_tools_debug: print('waiting for RS %s' % (i,)) else: kill_all_members() raise Exception( "Replica set still not initalized after %s minutes" % patience) return primary, replica_set_name def create_sharded_cluster(num_routers=3): global cur_port if not os.path.exists(logpath): os.makedirs(logpath) # Start a config server configdb_host = '%s:%d' % (hostname, cur_port) path = os.path.join(dbpath, 'configdb') if not os.path.exists(path): os.makedirs(path) configdb_logpath = os.path.join(logpath, 'configdb.log') cmd = [mongod, '--dbpath', path, '--port', str(cur_port), '--nojournal', '--logappend', '--logpath', configdb_logpath] proc = start_subprocess(cmd) nodes[configdb_host] = {'proc': proc, 'cmd': cmd, 'dbpath': path} res = wait_for(proc, cur_port) if not res: return None # ...and a shard server cur_port = cur_port + 1 shard_host = '%s:%d' % (hostname, cur_port) path = os.path.join(dbpath, 'shard1') if not os.path.exists(path): os.makedirs(path) db_logpath = os.path.join(logpath, 'shard1.log') cmd = [mongod, '--dbpath', path, '--port', str(cur_port), '--nojournal', '--logappend', '--logpath', db_logpath] proc = start_subprocess(cmd) nodes[shard_host] = {'proc': proc, 'cmd': cmd, 'dbpath': path} res = wait_for(proc, cur_port) if not res: return None # ...and a few mongos instances cur_port = cur_port + 1 for i in range(num_routers): cur_port = cur_port + i host = '%s:%d' % (hostname, cur_port) mongos_logpath = os.path.join(logpath, 'mongos' + str(i) + '.log') cmd = [mongos, '--port', str(cur_port), '--logappend', '--logpath', mongos_logpath, '--configdb', configdb_host] proc = start_subprocess(cmd) routers[host] = {'proc': proc, 'cmd': cmd} res = wait_for(proc, cur_port) if not res: return None # Add the shard client = pymongo.MongoClient(host) try: client.admin.command({'addshard': shard_host}) except pymongo.errors.OperationFailure: # Already configured. pass return get_mongos_seed_list() # Connect to a random member def get_client(): # Attempt a direct connection to each node until one succeeds. Using a # non-PRIMARY read preference allows us to use the node even if it's a # secondary. for i, node in enumerate(nodes.keys()): try: return connected( pymongo.MongoClient( node, read_preference=ReadPreference.PRIMARY_PREFERRED)) except pymongo.errors.ConnectionFailure: if i == len(nodes) - 1: raise def get_mongos_seed_list(): members = routers.keys() return ','.join(members) def kill_mongos(host): kill_members([host], 2, hosts=routers) return host def restart_mongos(host): restart_members([host], True) def get_members_in_state(state): status = get_client().admin.command('replSetGetStatus') members = status['members'] return [k['name'] for k in members if k['state'] == state] def get_primary(): try: primaries = get_members_in_state(1) assert len(primaries) <= 1 if primaries: return primaries[0] except (pymongo.errors.ConnectionFailure, pymongo.errors.OperationFailure): pass return None def wait_for_primary(): for _ in range(30): time.sleep(1) if get_primary(): break else: raise AssertionError("Primary didn't come back up") def get_random_secondary(): secondaries = get_members_in_state(2) if len(secondaries): return random.choice(secondaries) return None def get_secondaries(): return get_members_in_state(2) def get_arbiters(): return get_members_in_state(7) def get_recovering(): return get_members_in_state(3) def get_passives(): return get_client().admin.command('ismaster').get('passives', []) def get_hosts(): return get_client().admin.command('ismaster').get('hosts', []) def get_hidden_members(): # Both 'hidden' and 'slaveDelay' secondaries = get_secondaries() readers = get_hosts() + get_passives() for member in readers: try: secondaries.remove(member) except: # Skip primary pass return secondaries def get_tags(member): config = get_client().local.system.replset.find_one() for m in config['members']: if m['host'] == member: return m.get('tags', {}) raise Exception('member %s not in config' % repr(member)) def kill_primary(sig=2): primary = get_primary() kill_members([primary], sig) return primary def kill_secondary(sig=2): secondary = get_random_secondary() kill_members([secondary], sig) return secondary def kill_all_secondaries(sig=2): secondaries = get_secondaries() kill_members(secondaries, sig) return secondaries # TODO: refactor w/ start_replica_set def add_member(auth=False): global cur_port host = '%s:%d' % (hostname, cur_port) primary = get_primary() assert primary c = pymongo.MongoClient(primary) config = c.local.system.replset.find_one() _id = max([member['_id'] for member in config['members']]) + 1 member = {'_id': _id, 'host': host} path = os.path.join(dbpath, 'db' + str(_id)) if os.path.exists(path): shutil.rmtree(path) os.makedirs(path) member_logpath = os.path.join(logpath, 'db' + str(_id) + '.log') if not os.path.exists(os.path.dirname(member_logpath)): os.makedirs(os.path.dirname(member_logpath)) cmd = [mongod, '--dbpath', path, '--port', str(cur_port), '--replSet', replica_set_name, '--nojournal', '--oplogSize', '64', '--logappend', '--logpath', member_logpath] if auth: cmd += ['--keyFile', key_file] if ha_tools_debug: print('starting %s' % ' '.join(cmd)) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) nodes[host] = {'proc': proc, 'cmd': cmd, 'dbpath': path} res = wait_for(proc, cur_port) cur_port += 1 config['members'].append(member) config['version'] += 1 if ha_tools_debug: print({'replSetReconfig': config}) response = c.admin.command({'replSetReconfig': config}) if ha_tools_debug: print(response) if not res: return None return host def stepdown_primary(): primary = get_primary() if primary: if ha_tools_debug: print('stepping down primary: %s' % (primary,)) c = pymongo.MongoClient(primary) for _ in range(10): try: c.admin.command('replSetStepDown', 20) except pymongo.errors.OperationFailure as exc: if ha_tools_debug: print('Code %s from replSetStepDown: %s' % (exc.code, exc)) print('Trying again in one second....') time.sleep(1) except pymongo.errors.ConnectionFailure as exc: # replSetStepDown causes mongod to close all connections. if ha_tools_debug: print('Exception from replSetStepDown: %s' % exc) # Seems to have succeeded. break else: raise AssertionError("Couldn't complete replSetStepDown") if ha_tools_debug: print('\tcalled replSetStepDown') elif ha_tools_debug: print('stepdown_primary() found no primary') def set_maintenance(member, value): """Put a member into RECOVERING state if value is True, else normal state. """ c = pymongo.MongoClient(member) c.admin.command('replSetMaintenance', value) start = time.time() while value != (member in get_recovering()): assert (time.time() - start) <= 10, ( "Member %s never switched state" % member) time.sleep(0.25) def restart_members(members, router=False): restarted = [] for member in members: if router: cmd = routers[member]['cmd'] else: cmd = nodes[member]['cmd'] lockfile_path = os.path.join(nodes[member]['dbpath'], 'mongod.lock') if os.path.exists(lockfile_path): os.remove(lockfile_path) proc = start_subprocess(cmd) if router: routers[member]['proc'] = proc else: nodes[member]['proc'] = proc res = wait_for(proc, int(member.split(':')[1])) if res: restarted.append(member) return restarted pymongo-3.6.1/test/test_uri_parser.py0000644000076600000240000005434413245621354020215 0ustar shanestaff00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the pymongo uri_parser module.""" import copy import sys import warnings sys.path[0:0] = [""] from pymongo.uri_parser import (_partition, _rpartition, parse_userinfo, split_hosts, split_options, parse_uri) from pymongo.errors import ConfigurationError, InvalidURI from pymongo import ReadPreference from bson.binary import JAVA_LEGACY from bson.py3compat import string_type, _unicode from test import unittest class TestURI(unittest.TestCase): def test_partition(self): self.assertEqual(('foo', ':', 'bar'), _partition('foo:bar', ':')) self.assertEqual(('foobar', '', ''), _partition('foobar', ':')) def test_rpartition(self): self.assertEqual(('fo:o:', ':', 'bar'), _rpartition('fo:o::bar', ':')) self.assertEqual(('', '', 'foobar'), _rpartition('foobar', ':')) def test_validate_userinfo(self): self.assertRaises(InvalidURI, parse_userinfo, 'foo@') self.assertRaises(InvalidURI, parse_userinfo, ':password') self.assertRaises(InvalidURI, parse_userinfo, 'fo::o:p@ssword') self.assertRaises(InvalidURI, parse_userinfo, ':') self.assertTrue(parse_userinfo('user:password')) self.assertEqual(('us:r', 'p@ssword'), parse_userinfo('us%3Ar:p%40ssword')) self.assertEqual(('us er', 'p ssword'), parse_userinfo('us+er:p+ssword')) self.assertEqual(('us er', 'p ssword'), parse_userinfo('us%20er:p%20ssword')) self.assertEqual(('us+er', 'p+ssword'), parse_userinfo('us%2Ber:p%2Bssword')) self.assertEqual(('dev1@FOO.COM', ''), parse_userinfo('dev1%40FOO.COM')) self.assertEqual(('dev1@FOO.COM', ''), parse_userinfo('dev1%40FOO.COM:')) def test_split_hosts(self): self.assertRaises(ConfigurationError, split_hosts, 'localhost:27017,') self.assertRaises(ConfigurationError, split_hosts, ',localhost:27017') self.assertRaises(ConfigurationError, split_hosts, 'localhost:27017,,localhost:27018') self.assertEqual([('localhost', 27017), ('example.com', 27017)], split_hosts('localhost,example.com')) self.assertEqual([('localhost', 27018), ('example.com', 27019)], split_hosts('localhost:27018,example.com:27019')) self.assertEqual([('/tmp/mongodb-27017.sock', None)], split_hosts('/tmp/mongodb-27017.sock')) self.assertEqual([('/tmp/mongodb-27017.sock', None), ('example.com', 27017)], split_hosts('/tmp/mongodb-27017.sock,' 'example.com:27017')) self.assertEqual([('example.com', 27017), ('/tmp/mongodb-27017.sock', None)], split_hosts('example.com:27017,' '/tmp/mongodb-27017.sock')) self.assertRaises(ValueError, split_hosts, '::1', 27017) self.assertRaises(ValueError, split_hosts, '[::1:27017') self.assertRaises(ValueError, split_hosts, '::1') self.assertRaises(ValueError, split_hosts, '::1]:27017') self.assertEqual([('::1', 27017)], split_hosts('[::1]:27017')) self.assertEqual([('::1', 27017)], split_hosts('[::1]')) def test_split_options(self): self.assertRaises(ConfigurationError, split_options, 'foo') self.assertRaises(ConfigurationError, split_options, 'foo=bar;foo') self.assertTrue(split_options('ssl=true')) self.assertTrue(split_options('connect=true')) self.assertTrue(split_options('ssl_match_hostname=true')) # Test Invalid URI options that should throw warnings. with warnings.catch_warnings(): warnings.filterwarnings('error') self.assertRaises(Warning, split_options, 'foo=bar', warn=True) self.assertRaises(Warning, split_options, 'socketTimeoutMS=foo', warn=True) self.assertRaises(Warning, split_options, 'socketTimeoutMS=0.0', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=foo', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=0.0', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=1e100000', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=-1e100000', warn=True) self.assertRaises(Warning, split_options, 'ssl=foo', warn=True) self.assertRaises(Warning, split_options, 'connect=foo', warn=True) self.assertRaises(Warning, split_options, 'ssl_match_hostname=foo', warn=True) # On most platforms float('inf') and float('-inf') represent # +/- infinity, although on Python 2.4 and 2.5 on Windows those # expressions are invalid if not (sys.platform == "win32" and sys.version_info <= (2, 5)): self.assertRaises(Warning, split_options, 'connectTimeoutMS=inf', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=-inf', warn=True) self.assertRaises(Warning, split_options, 'wtimeoutms=foo', warn=True) self.assertRaises(Warning, split_options, 'wtimeoutms=5.5', warn=True) self.assertRaises(Warning, split_options, 'fsync=foo', warn=True) self.assertRaises(Warning, split_options, 'fsync=5.5', warn=True) self.assertRaises(Warning, split_options, 'authMechanism=foo', warn=True) # Test invalid options with warn=False. self.assertRaises(ConfigurationError, split_options, 'foo=bar') self.assertRaises(ValueError, split_options, 'socketTimeoutMS=foo') self.assertRaises(ValueError, split_options, 'socketTimeoutMS=0.0') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=foo') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=0.0') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=1e100000') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=-1e100000') self.assertRaises(ValueError, split_options, 'ssl=foo') self.assertRaises(ValueError, split_options, 'connect=foo') self.assertRaises(ValueError, split_options, 'ssl_match_hostname=foo') if not (sys.platform == "win32" and sys.version_info <= (2, 5)): self.assertRaises(ValueError, split_options, 'connectTimeoutMS=inf') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=-inf') self.assertRaises(ValueError, split_options, 'wtimeoutms=foo') self.assertRaises(ValueError, split_options, 'wtimeoutms=5.5') self.assertRaises(ValueError, split_options, 'fsync=foo') self.assertRaises(ValueError, split_options, 'fsync=5.5') self.assertRaises(ValueError, split_options, 'authMechanism=foo') # Test splitting options works when valid. self.assertTrue(split_options('socketTimeoutMS=300')) self.assertTrue(split_options('connectTimeoutMS=300')) self.assertEqual({'sockettimeoutms': 0.3}, split_options('socketTimeoutMS=300')) self.assertEqual({'sockettimeoutms': 0.0001}, split_options('socketTimeoutMS=0.1')) self.assertEqual({'connecttimeoutms': 0.3}, split_options('connectTimeoutMS=300')) self.assertEqual({'connecttimeoutms': 0.0001}, split_options('connectTimeoutMS=0.1')) self.assertTrue(split_options('connectTimeoutMS=300')) self.assertTrue(isinstance(split_options('w=5')['w'], int)) self.assertTrue(isinstance(split_options('w=5.5')['w'], string_type)) self.assertTrue(split_options('w=foo')) self.assertTrue(split_options('w=majority')) self.assertTrue(split_options('wtimeoutms=500')) self.assertEqual({'fsync': True}, split_options('fsync=true')) self.assertEqual({'fsync': False}, split_options('fsync=false')) self.assertEqual({'authmechanism': 'GSSAPI'}, split_options('authMechanism=GSSAPI')) self.assertEqual({'authmechanism': 'MONGODB-CR'}, split_options('authMechanism=MONGODB-CR')) self.assertEqual({'authmechanism': 'SCRAM-SHA-1'}, split_options('authMechanism=SCRAM-SHA-1')) self.assertEqual({'authsource': 'foobar'}, split_options('authSource=foobar')) self.assertEqual({'maxpoolsize': 50}, split_options('maxpoolsize=50')) def test_parse_uri(self): self.assertRaises(InvalidURI, parse_uri, "http://foobar.com") self.assertRaises(InvalidURI, parse_uri, "http://foo@foobar.com") self.assertRaises(ValueError, parse_uri, "mongodb://::1", 27017) orig = { 'nodelist': [("localhost", 27017)], 'username': None, 'password': None, 'database': None, 'collection': None, 'options': {} } res = copy.deepcopy(orig) self.assertEqual(res, parse_uri("mongodb://localhost")) res.update({'username': 'fred', 'password': 'foobar'}) self.assertEqual(res, parse_uri("mongodb://fred:foobar@localhost")) res.update({'database': 'baz'}) self.assertEqual(res, parse_uri("mongodb://fred:foobar@localhost/baz")) res = copy.deepcopy(orig) res['nodelist'] = [("example1.com", 27017), ("example2.com", 27017)] self.assertEqual(res, parse_uri("mongodb://example1.com:27017," "example2.com:27017")) res = copy.deepcopy(orig) res['nodelist'] = [("localhost", 27017), ("localhost", 27018), ("localhost", 27019)] self.assertEqual(res, parse_uri("mongodb://localhost," "localhost:27018,localhost:27019")) res = copy.deepcopy(orig) res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://localhost/foo")) res = copy.deepcopy(orig) self.assertEqual(res, parse_uri("mongodb://localhost/")) res.update({'database': 'test', 'collection': 'yield_historical.in'}) self.assertEqual(res, parse_uri("mongodb://" "localhost/test.yield_historical.in")) res.update({'username': 'fred', 'password': 'foobar'}) self.assertEqual(res, parse_uri("mongodb://fred:foobar@localhost/" "test.yield_historical.in")) res = copy.deepcopy(orig) res['nodelist'] = [("example1.com", 27017), ("example2.com", 27017)] res.update({'database': 'test', 'collection': 'yield_historical.in'}) self.assertEqual(res, parse_uri("mongodb://example1.com:27017,example2.com" ":27017/test.yield_historical.in")) # Test socket path without escaped characters. self.assertRaises(InvalidURI, parse_uri, "mongodb:///tmp/mongodb-27017.sock") # Test with escaped characters. res = copy.deepcopy(orig) res['nodelist'] = [("example2.com", 27017), ("/tmp/mongodb-27017.sock", None)] self.assertEqual(res, parse_uri("mongodb://example2.com," "%2Ftmp%2Fmongodb-27017.sock")) res = copy.deepcopy(orig) res['nodelist'] = [("shoe.sock.pants.co.uk", 27017), ("/tmp/mongodb-27017.sock", None)] res['database'] = "nethers_db" self.assertEqual(res, parse_uri("mongodb://shoe.sock.pants.co.uk," "%2Ftmp%2Fmongodb-27017.sock/nethers_db")) res = copy.deepcopy(orig) res['nodelist'] = [("/tmp/mongodb-27017.sock", None), ("example2.com", 27017)] res.update({'database': 'test', 'collection': 'yield_historical.in'}) self.assertEqual(res, parse_uri("mongodb://%2Ftmp%2Fmongodb-27017.sock," "example2.com:27017" "/test.yield_historical.in")) res = copy.deepcopy(orig) res['nodelist'] = [("/tmp/mongodb-27017.sock", None), ("example2.com", 27017)] res.update({'database': 'test', 'collection': 'yield_historical.sock'}) self.assertEqual(res, parse_uri("mongodb://%2Ftmp%2Fmongodb-27017.sock," "example2.com:27017/test.yield_historical" ".sock")) res = copy.deepcopy(orig) res['nodelist'] = [("example2.com", 27017)] res.update({'database': 'test', 'collection': 'yield_historical.sock'}) self.assertEqual(res, parse_uri("mongodb://example2.com:27017" "/test.yield_historical.sock")) res = copy.deepcopy(orig) res['nodelist'] = [("/tmp/mongodb-27017.sock", None)] res.update({'database': 'test', 'collection': 'mongodb-27017.sock'}) self.assertEqual(res, parse_uri("mongodb://%2Ftmp%2Fmongodb-27017.sock" "/test.mongodb-27017.sock")) res = copy.deepcopy(orig) res['nodelist'] = [('/tmp/mongodb-27020.sock', None), ("::1", 27017), ("2001:0db8:85a3:0000:0000:8a2e:0370:7334", 27018), ("192.168.0.212", 27019), ("localhost", 27018)] self.assertEqual(res, parse_uri("mongodb://%2Ftmp%2Fmongodb-27020.sock" ",[::1]:27017,[2001:0db8:" "85a3:0000:0000:8a2e:0370:7334]," "192.168.0.212:27019,localhost", 27018)) res = copy.deepcopy(orig) res.update({'username': 'fred', 'password': 'foobar'}) res.update({'database': 'test', 'collection': 'yield_historical.in'}) self.assertEqual(res, parse_uri("mongodb://fred:foobar@localhost/" "test.yield_historical.in")) res = copy.deepcopy(orig) res['database'] = 'test' res['collection'] = 'name/with "delimiters' self.assertEqual( res, parse_uri("mongodb://localhost/test.name/with \"delimiters")) res = copy.deepcopy(orig) res['options'] = { 'readpreference': ReadPreference.SECONDARY.mongos_mode } self.assertEqual(res, parse_uri( "mongodb://localhost/?readPreference=secondary")) # Various authentication tests res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'MONGODB-CR'} res['username'] = 'user' res['password'] = 'password' self.assertEqual(res, parse_uri("mongodb://user:password@localhost/" "?authMechanism=MONGODB-CR")) res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'MONGODB-CR', 'authsource': 'bar'} res['username'] = 'user' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user:password@localhost/foo" "?authSource=bar;authMechanism=MONGODB-CR")) res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'MONGODB-CR'} res['username'] = 'user' res['password'] = '' self.assertEqual(res, parse_uri("mongodb://user:@localhost/" "?authMechanism=MONGODB-CR")) res = copy.deepcopy(orig) res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo")) res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'GSSAPI'} res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo?authMechanism=GSSAPI")) res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'GSSAPI'} res['username'] = 'user@domain.com' res['password'] = '' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com" "@localhost/foo?authMechanism=GSSAPI")) res = copy.deepcopy(orig) res['options'] = { 'readpreference': ReadPreference.SECONDARY.mongos_mode, 'readpreferencetags': [ {'dc': 'west', 'use': 'website'}, {'dc': 'east', 'use': 'website'} ] } res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo?readpreference=secondary&" "readpreferencetags=dc:west,use:website&" "readpreferencetags=dc:east,use:website")) res = copy.deepcopy(orig) res['options'] = { 'readpreference': ReadPreference.SECONDARY.mongos_mode, 'readpreferencetags': [ {'dc': 'west', 'use': 'website'}, {'dc': 'east', 'use': 'website'}, {} ] } res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo?readpreference=secondary&" "readpreferencetags=dc:west,use:website&" "readpreferencetags=dc:east,use:website&" "readpreferencetags=")) res = copy.deepcopy(orig) res['options'] = {'uuidrepresentation': JAVA_LEGACY} res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo?uuidrepresentation=" "javaLegacy")) with warnings.catch_warnings(): warnings.filterwarnings('error') self.assertRaises(Warning, parse_uri, "mongodb://user%40domain.com:password" "@localhost/foo?uuidrepresentation=notAnOption", warn=True) self.assertRaises(ValueError, parse_uri, "mongodb://user%40domain.com:password" "@localhost/foo?uuidrepresentation=notAnOption") def test_parse_uri_unicode(self): # Ensure parsing a unicode returns option names that can be passed # as kwargs. In Python 2.4, keyword argument names must be ASCII. # In all Pythons, str is the type of valid keyword arg names. res = parse_uri(_unicode("mongodb://localhost/?fsync=true")) for key in res['options']: self.assertTrue(isinstance(key, str)) def test_parse_ssl_paths(self): # Turn off "validate" since these paths don't exist on filesystem. self.assertEqual( {'collection': None, 'database': None, 'nodelist': [('/MongoDB.sock', None)], 'options': {'ssl_certfile': '/a/b'}, 'password': 'foo/bar', 'username': 'jesse'}, parse_uri( 'mongodb://jesse:foo%2Fbar@%2FMongoDB.sock/?ssl_certfile=/a/b', validate=False)) self.assertEqual( {'collection': None, 'database': None, 'nodelist': [('/MongoDB.sock', None)], 'options': {'ssl_certfile': 'a/b'}, 'password': 'foo/bar', 'username': 'jesse'}, parse_uri( 'mongodb://jesse:foo%2Fbar@%2FMongoDB.sock/?ssl_certfile=a/b', validate=False)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_session.py0000644000076600000240000013500513245621354017517 0ustar shanestaff00000000000000# Copyright 2017 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the client_session module.""" import copy import sys from bson import DBRef from bson.py3compat import StringIO from gridfs import GridFS, GridFSBucket from pymongo import ASCENDING, InsertOne, IndexModel, OFF, monitoring from pymongo.common import _MAX_END_SESSIONS from pymongo.errors import (ConfigurationError, InvalidOperation, OperationFailure) from pymongo.monotonic import time as _time from pymongo.read_concern import ReadConcern from pymongo.write_concern import WriteConcern from test import IntegrationTest, client_context, db_user, db_pwd, unittest, SkipTest from test.utils import ignore_deprecations, rs_or_single_client, EventListener # Ignore auth commands like saslStart, so we can assert lsid is in all commands. class SessionTestListener(EventListener): def started(self, event): if not event.command_name.startswith('sasl'): super(SessionTestListener, self).started(event) def succeeded(self, event): if not event.command_name.startswith('sasl'): super(SessionTestListener, self).succeeded(event) def failed(self, event): if not event.command_name.startswith('sasl'): super(SessionTestListener, self).failed(event) def first_command_started(self): assert len(self.results['started']) >= 1, ( "No command-started events") return self.results['started'][0] def session_ids(client): return [s.session_id for s in copy.copy(client._topology._session_pool)] class TestSession(IntegrationTest): @classmethod @client_context.require_sessions def setUpClass(cls): super(TestSession, cls).setUpClass() # Create a second client so we can make sure clients cannot share # sessions. cls.client2 = rs_or_single_client() # Redact no commands, so we can test user-admin commands have "lsid". cls.sensitive_commands = monitoring._SENSITIVE_COMMANDS.copy() monitoring._SENSITIVE_COMMANDS.clear() @classmethod def tearDownClass(cls): monitoring._SENSITIVE_COMMANDS.update(cls.sensitive_commands) super(TestSession, cls).tearDownClass() def setUp(self): self.listener = SessionTestListener() self.session_checker_listener = SessionTestListener() self.client = rs_or_single_client( event_listeners=[self.listener, self.session_checker_listener]) self.db = self.client.pymongo_test self.initial_lsids = set(s['id'] for s in session_ids(self.client)) def tearDown(self): """All sessions used in the test must be returned to the pool.""" self.client.drop_database('pymongo_test') used_lsids = self.initial_lsids.copy() for event in self.session_checker_listener.results['started']: if 'lsid' in event.command: used_lsids.add(event.command['lsid']['id']) current_lsids = set(s['id'] for s in session_ids(self.client)) self.assertLessEqual(used_lsids, current_lsids) def _test_ops(self, client, *ops): listener = client.event_listeners()[0][0] for f, args, kw in ops: with client.start_session() as s: last_use = s._server_session.last_use start = _time() self.assertLessEqual(last_use, start) listener.results.clear() # In case "f" modifies its inputs. args = copy.copy(args) kw = copy.copy(kw) kw['session'] = s f(*args, **kw) self.assertGreaterEqual(s._server_session.last_use, start) self.assertGreaterEqual(len(listener.results['started']), 1) for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "%s sent no lsid with %s" % ( f.__name__, event.command_name)) self.assertEqual( s.session_id, event.command['lsid'], "%s sent wrong lsid with %s" % ( f.__name__, event.command_name)) self.assertFalse(s.has_ended) self.assertTrue(s.has_ended) with self.assertRaisesRegex(InvalidOperation, "ended session"): f(*args, **kw) # Test a session cannot be used on another client. with self.client2.start_session() as s: # In case "f" modifies its inputs. args = copy.copy(args) kw = copy.copy(kw) kw['session'] = s with self.assertRaisesRegex( InvalidOperation, 'Can only use session with the MongoClient' ' that started it'): f(*args, **kw) # No explicit session. for f, args, kw in ops: listener.results.clear() f(*args, **kw) self.assertGreaterEqual(len(listener.results['started']), 1) lsids = [] for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "%s sent no lsid with %s" % ( f.__name__, event.command_name)) lsids.append(event.command['lsid']) if not (sys.platform.startswith('java') or 'PyPy' in sys.version): # Server session was returned to pool. Ignore interpreters with # non-deterministic GC. for lsid in lsids: self.assertIn( lsid, session_ids(client), "%s did not return implicit session to pool" % ( f.__name__,)) def test_pool_lifo(self): # "Pool is LIFO" test from Driver Sessions Spec. a = self.client.start_session() b = self.client.start_session() a_id = a.session_id b_id = b.session_id a.end_session() b.end_session() s = self.client.start_session() self.assertEqual(b_id, s.session_id) self.assertNotEqual(a_id, s.session_id) s2 = self.client.start_session() self.assertEqual(a_id, s2.session_id) self.assertNotEqual(b_id, s2.session_id) s.end_session() s2.end_session() def test_end_session(self): # We test elsewhere that using an ended session throws InvalidOperation. client = self.client s = client.start_session() self.assertFalse(s.has_ended) self.assertIsNotNone(s.session_id) s.end_session() self.assertTrue(s.has_ended) with self.assertRaisesRegex(InvalidOperation, "ended session"): s.session_id def test_end_sessions(self): # Use a new client so that the tearDown hook does not error. listener = SessionTestListener() client = rs_or_single_client(event_listeners=[listener]) # Start many sessions. sessions = [client.start_session() for _ in range(_MAX_END_SESSIONS + 1)] for s in sessions: s.end_session() # Closing the client should end all sessions and clear the pool. self.assertEqual(len(client._topology._session_pool), _MAX_END_SESSIONS + 1) client.close() self.assertEqual(len(client._topology._session_pool), 0) end_sessions = [e for e in listener.results['started'] if e.command_name == 'endSessions'] self.assertEqual(len(end_sessions), 2) # Closing again should not send any commands. listener.results.clear() client.close() self.assertEqual(len(listener.results['started']), 0) def test_client(self): client = self.client # Make sure if the test fails we unlock the server. def unlock(): try: client.unlock() except OperationFailure: pass self.addCleanup(unlock) ops = [ (client.server_info, [], {}), (client.database_names, [], {}), (client.drop_database, ['pymongo_test'], {}), ] if not client_context.is_mongos: ops.extend([ (client.fsync, [], {'lock': True}), (client.unlock, [], {}), ]) self._test_ops(client, *ops) def test_database(self): client = self.client db = client.pymongo_test ops = [ (db.command, ['ping'], {}), (db.create_collection, ['collection'], {}), (db.collection_names, [], {}), (db.validate_collection, ['collection'], {}), (db.drop_collection, ['collection'], {}), (db.current_op, [], {}), (db.profiling_info, [], {}), (db.dereference, [DBRef('collection', 1)], {}), ] if not client_context.is_mongos: ops.append((db.set_profiling_level, [OFF], {})) ops.append((db.profiling_level, [], {})) self._test_ops(client, *ops) @client_context.require_auth @ignore_deprecations def test_user_admin(self): client = self.client db = client.pymongo_test extra = {'roles': ['read']} if client_context.version.at_least(3, 7, 2): extra['mechanisms'] = ['SCRAM-SHA-1'] self._test_ops( client, (db.add_user, ['session-test', 'pass'], extra), # Do it again to test updateUser command. (db.add_user, ['session-test', 'pass'], extra), (db.remove_user, ['session-test'], {})) def test_collection(self): client = self.client coll = client.pymongo_test.collection # Test some collection methods - the rest are in test_cursor. self._test_ops( client, (coll.drop, [], {}), (coll.bulk_write, [[InsertOne({})]], {}), (coll.insert_one, [{}], {}), (coll.insert_many, [[{}, {}]], {}), (coll.replace_one, [{}, {}], {}), (coll.update_one, [{}, {'$set': {'a': 1}}], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_one, [{}], {}), (coll.delete_many, [{}], {}), (coll.map_reduce, ['function() {}', 'function() {}', 'output'], {}), (coll.inline_map_reduce, ['function() {}', 'function() {}'], {}), (coll.find_one_and_replace, [{}, {}], {}), (coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}), (coll.find_one_and_delete, [{}, {}], {}), (coll.rename, ['collection2'], {}), # Drop collection2 between tests of "rename", above. (client.pymongo_test.drop_collection, ['collection2'], {}), (coll.distinct, ['a'], {}), (coll.find_one, [], {}), (coll.count, [], {}), (coll.create_indexes, [[IndexModel('a')]], {}), (coll.create_index, ['a'], {}), (coll.drop_index, ['a_1'], {}), (coll.drop_indexes, [], {}), (coll.reindex, [], {}), (coll.list_indexes, [], {}), (coll.index_information, [], {}), (coll.options, [], {}), (coll.aggregate, [[]], {})) @client_context.require_no_mongos def test_parallel_collection_scan(self): listener = self.listener client = self.client coll = client.pymongo_test.collection coll.insert_many([{'_id': i} for i in range(1000)]) listener.results.clear() def scan(session=None): cursors = coll.parallel_scan(4, session=session) for c in cursors: c.batch_size(2) list(c) self._test_ops(client, (scan, [], {})) # Implicit session with parallel_scan is uncorrelated with cursors', # but each cursor's getMores all use the same lsid. listener.results.clear() scan() cursor_lsids = {} for event in listener.results['started']: self.assertIn( 'lsid', event.command, "parallel_scan sent no lsid with %s" % (event.command_name, )) if event.command_name == 'getMore': cursor_id = event.command['getMore'] if cursor_id in cursor_lsids: self.assertEqual(cursor_lsids[cursor_id], event.command['lsid']) else: cursor_lsids[cursor_id] = event.command['lsid'] def test_cursor_clone(self): coll = self.client.pymongo_test.collection # Ensure some batches. coll.insert_many({} for _ in range(10)) self.addCleanup(coll.drop) with self.client.start_session() as s: cursor = coll.find(session=s) self.assertTrue(cursor.session is s) clone = cursor.clone() self.assertTrue(clone.session is s) # No explicit session. cursor = coll.find(batch_size=2) next(cursor) # Session is "owned" by cursor. self.assertIsNone(cursor.session) self.assertIsNotNone(cursor._Cursor__session) clone = cursor.clone() next(clone) self.assertIsNone(clone.session) self.assertIsNotNone(clone._Cursor__session) self.assertFalse(cursor._Cursor__session is clone._Cursor__session) cursor.close() clone.close() def test_cursor(self): listener = self.listener client = self.client coll = client.pymongo_test.collection coll.insert_many([{} for _ in range(1000)]) # Test all cursor methods. ops = [ ('find', lambda session: list(coll.find(session=session))), ('getitem', lambda session: coll.find(session=session)[0]), ('count', lambda session: coll.find(session=session).count()), ('distinct', lambda session: coll.find(session=session).distinct('a')), ('explain', lambda session: coll.find(session=session).explain()), ] for name, f in ops: with client.start_session() as s: listener.results.clear() f(session=s) self.assertGreaterEqual(len(listener.results['started']), 1) for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "%s sent no lsid with %s" % ( name, event.command_name)) self.assertEqual( s.session_id, event.command['lsid'], "%s sent wrong lsid with %s" % ( name, event.command_name)) with self.assertRaisesRegex(InvalidOperation, "ended session"): f(session=s) # No explicit session. for name, f in ops: listener.results.clear() f(session=None) event0 = listener.first_command_started() self.assertTrue( 'lsid' in event0.command, "%s sent no lsid with %s" % ( name, event0.command_name)) lsid = event0.command['lsid'] for event in listener.results['started'][1:]: self.assertTrue( 'lsid' in event.command, "%s sent no lsid with %s" % ( name, event.command_name)) self.assertEqual( lsid, event.command['lsid'], "%s sent wrong lsid with %s" % ( name, event.command_name)) def test_gridfs(self): client = self.client fs = GridFS(client.pymongo_test) def new_file(session=None): grid_file = fs.new_file(_id=1, filename='f', session=session) # 1 MB, 5 chunks, to test that each chunk is fetched with same lsid. grid_file.write(b'a' * 1048576) grid_file.close() def find(session=None): files = list(fs.find({'_id': 1}, session=session)) for f in files: f.read() self._test_ops( client, (new_file, [], {}), (fs.put, [b'data'], {}), (lambda session=None: fs.get(1, session=session).read(), [], {}), (lambda session=None: fs.get_version('f', session=session).read(), [], {}), (lambda session=None: fs.get_last_version('f', session=session).read(), [], {}), (fs.list, [], {}), (fs.find_one, [1], {}), (lambda session=None: list(fs.find(session=session)), [], {}), (fs.exists, [1], {}), (find, [], {}), (fs.delete, [1], {})) def test_gridfs_bucket(self): client = self.client bucket = GridFSBucket(client.pymongo_test) def upload(session=None): stream = bucket.open_upload_stream('f', session=session) stream.write(b'a' * 1048576) stream.close() def upload_with_id(session=None): stream = bucket.open_upload_stream_with_id(1, 'f1', session=session) stream.write(b'a' * 1048576) stream.close() def open_download_stream(session=None): stream = bucket.open_download_stream(1, session=session) stream.read() def open_download_stream_by_name(session=None): stream = bucket.open_download_stream_by_name('f', session=session) stream.read() def find(session=None): files = list(bucket.find({'_id': 1}, session=session)) for f in files: f.read() sio = StringIO() self._test_ops( client, (upload, [], {}), (upload_with_id, [], {}), (bucket.upload_from_stream, ['f', b'data'], {}), (bucket.upload_from_stream_with_id, [2, 'f', b'data'], {}), (open_download_stream, [], {}), (open_download_stream_by_name, [], {}), (bucket.download_to_stream, [1, sio], {}), (bucket.download_to_stream_by_name, ['f', sio], {}), (find, [], {}), (bucket.rename, [1, 'f2'], {}), # Delete both files so _test_ops can run these operations twice. (bucket.delete, [1], {}), (bucket.delete, [2], {})) def test_gridfsbucket_cursor(self): client = self.client bucket = GridFSBucket(client.pymongo_test) for file_id in 1, 2: stream = bucket.open_upload_stream_with_id(file_id, str(file_id)) stream.write(b'a' * 1048576) stream.close() with client.start_session() as s: cursor = bucket.find(session=s) for f in cursor: f.read() self.assertFalse(s.has_ended) self.assertTrue(s.has_ended) # No explicit session. cursor = bucket.find(batch_size=1) files = [cursor.next()] s = cursor._Cursor__session self.assertFalse(s.has_ended) cursor.__del__() self.assertTrue(s.has_ended) self.assertIsNone(cursor._Cursor__session) # Files are still valid, they use their own sessions. for f in files: f.read() # Explicit session. with client.start_session() as s: cursor = bucket.find(session=s) s = cursor.session files = list(cursor) cursor.__del__() self.assertFalse(s.has_ended) for f in files: f.read() with self.assertRaisesRegex(InvalidOperation, "ended session"): files[0].read() def test_aggregate(self): client = self.client coll = client.pymongo_test.collection def agg(session=None): list(coll.aggregate( [], batchSize=2, session=session)) # With empty collection. self._test_ops(client, (agg, [], {})) # Now with documents. coll.insert_many([{} for _ in range(10)]) self.addCleanup(coll.drop) self._test_ops(client, (agg, [], {})) def test_killcursors(self): client = self.client coll = client.pymongo_test.collection coll.insert_many([{} for _ in range(10)]) def explicit_close(session=None): cursor = coll.find(batch_size=2, session=session) next(cursor) cursor.close() self._test_ops(client, (explicit_close, [], {})) def test_aggregate_error(self): listener = self.listener client = self.client coll = client.pymongo_test.collection # 3.6.0 mongos only validates the aggregate pipeline when the # database exists. coll.insert_one({}) listener.results.clear() with self.assertRaises(OperationFailure): coll.aggregate([{'$badOperation': {'bar': 1}}]) event = listener.first_command_started() self.assertEqual(event.command_name, 'aggregate') lsid = event.command['lsid'] # Session was returned to pool despite error. self.assertIn(lsid, session_ids(client)) def _test_cursor_helper(self, create_cursor, close_cursor): coll = self.client.pymongo_test.collection coll.insert_many([{} for _ in range(1000)]) cursor = create_cursor(coll, None) next(cursor) # Session is "owned" by cursor. session = getattr(cursor, '_%s__session' % cursor.__class__.__name__) self.assertIsNotNone(session) lsid = session.session_id next(cursor) # Cursor owns its session unto death. self.assertNotIn(lsid, session_ids(self.client)) close_cursor(cursor) self.assertIn(lsid, session_ids(self.client)) # An explicit session is not ended by cursor.close() or list(cursor). with self.client.start_session() as s: cursor = create_cursor(coll, s) next(cursor) close_cursor(cursor) self.assertFalse(s.has_ended) lsid = s.session_id self.assertTrue(s.has_ended) self.assertIn(lsid, session_ids(self.client)) def test_cursor_close(self): self._test_cursor_helper( lambda coll, session: coll.find(session=session), lambda cursor: cursor.close()) def test_command_cursor_close(self): self._test_cursor_helper( lambda coll, session: coll.aggregate([], session=session), lambda cursor: cursor.close()) def test_cursor_del(self): self._test_cursor_helper( lambda coll, session: coll.find(session=session), lambda cursor: cursor.__del__()) def test_command_cursor_del(self): self._test_cursor_helper( lambda coll, session: coll.aggregate([], session=session), lambda cursor: cursor.__del__()) def test_cursor_exhaust(self): self._test_cursor_helper( lambda coll, session: coll.find(session=session), lambda cursor: list(cursor)) def test_command_cursor_exhaust(self): self._test_cursor_helper( lambda coll, session: coll.aggregate([], session=session), lambda cursor: list(cursor)) def test_cursor_limit_reached(self): self._test_cursor_helper( lambda coll, session: coll.find(limit=4, batch_size=2, session=session), lambda cursor: list(cursor)) def test_command_cursor_limit_reached(self): self._test_cursor_helper( lambda coll, session: coll.aggregate([], batchSize=900, session=session), lambda cursor: list(cursor)) class TestCausalConsistency(unittest.TestCase): @classmethod def setUpClass(cls): cls.listener = SessionTestListener() cls.client = rs_or_single_client(event_listeners=[cls.listener]) @client_context.require_sessions def setUp(self): super(TestCausalConsistency, self).setUp() @client_context.require_no_standalone def test_core(self): with self.client.start_session() as sess: self.assertIsNone(sess.cluster_time) self.assertIsNone(sess.operation_time) self.listener.results.clear() self.client.pymongo_test.test.find_one(session=sess) started = self.listener.results['started'][0] cmd = started.command self.assertIsNone(cmd.get('readConcern')) op_time = sess.operation_time self.assertIsNotNone(op_time) succeeded = self.listener.results['succeeded'][0] reply = succeeded.reply self.assertEqual(op_time, reply.get('operationTime')) # No explicit session self.client.pymongo_test.test.insert_one({}) self.assertEqual(sess.operation_time, op_time) self.listener.results.clear() try: self.client.pymongo_test.command('doesntexist', session=sess) except: pass failed = self.listener.results['failed'][0] failed_op_time = failed.failure.get('operationTime') # Some older builds of MongoDB 3.5 / 3.6 return None for # operationTime when a command fails. Make sure we don't # change operation_time to None. if failed_op_time is None: self.assertIsNotNone(sess.operation_time) else: self.assertEqual( sess.operation_time, failed_op_time) with self.client.start_session() as sess2: self.assertIsNone(sess2.cluster_time) self.assertIsNone(sess2.operation_time) self.assertRaises(TypeError, sess2.advance_cluster_time, 1) self.assertRaises(ValueError, sess2.advance_cluster_time, {}) self.assertRaises(TypeError, sess2.advance_operation_time, 1) # No error sess2.advance_cluster_time(sess.cluster_time) sess2.advance_operation_time(sess.operation_time) self.assertEqual(sess.cluster_time, sess2.cluster_time) self.assertEqual(sess.operation_time, sess2.operation_time) def _test_reads(self, op): coll = self.client.pymongo_test.test with self.client.start_session() as sess: coll.find_one({}, session=sess) operation_time = sess.operation_time self.assertIsNotNone(operation_time) self.listener.results.clear() op(coll, sess) act = self.listener.results['started'][0].command.get( 'readConcern', {}).get('afterClusterTime') self.assertEqual(operation_time, act) @client_context.require_no_standalone def test_reads(self): # Make sure the collection exists. self.client.pymongo_test.test.insert_one({}) self._test_reads( lambda coll, session: list(coll.aggregate([], session=session))) self._test_reads( lambda coll, session: list(coll.find({}, session=session))) self._test_reads( lambda coll, session: coll.find_one({}, session=session)) self._test_reads( lambda coll, session: coll.count(session=session)) self._test_reads( lambda coll, session: coll.distinct('foo', session=session)) self._test_reads( lambda coll, session: coll.map_reduce( 'function() {}', 'function() {}', 'inline', session=session)) self._test_reads( lambda coll, session: coll.inline_map_reduce( 'function() {}', 'function() {}', session=session)) if not client_context.is_mongos: def scan(coll, session): cursors = coll.parallel_scan(1, session=session) for cur in cursors: list(cur) self._test_reads( lambda coll, session: scan(coll, session=session)) self.assertRaises( ConfigurationError, self._test_reads, lambda coll, session: list( coll.aggregate_raw_batches([], session=session))) self.assertRaises( ConfigurationError, self._test_reads, lambda coll, session: list( coll.find_raw_batches({}, session=session))) def _test_writes(self, op): coll = self.client.pymongo_test.test with self.client.start_session() as sess: op(coll, sess) operation_time = sess.operation_time self.assertIsNotNone(operation_time) self.listener.results.clear() coll.find_one({}, session=sess) act = self.listener.results['started'][0].command.get( 'readConcern', {}).get('afterClusterTime') self.assertEqual(operation_time, act) @client_context.require_no_standalone def test_writes(self): self._test_writes( lambda coll, session: coll.bulk_write( [InsertOne({})], session=session)) self._test_writes( lambda coll, session: coll.insert_one({}, session=session)) self._test_writes( lambda coll, session: coll.insert_many([{}], session=session)) self._test_writes( lambda coll, session: coll.replace_one( {'_id': 1}, {'x': 1}, session=session)) self._test_writes( lambda coll, session: coll.update_one( {}, {'$set': {'X': 1}}, session=session)) self._test_writes( lambda coll, session: coll.update_many( {}, {'$set': {'x': 1}}, session=session)) self._test_writes( lambda coll, session: coll.delete_one({}, session=session)) self._test_writes( lambda coll, session: coll.delete_many({}, session=session)) self._test_writes( lambda coll, session: coll.find_one_and_replace( {'x': 1}, {'y': 1}, session=session)) self._test_writes( lambda coll, session: coll.find_one_and_update( {'y': 1}, {'$set': {'x': 1}}, session=session)) self._test_writes( lambda coll, session: coll.find_one_and_delete( {'x': 1}, session=session)) self._test_writes( lambda coll, session: coll.create_index("foo", session=session)) self._test_writes( lambda coll, session: coll.create_indexes( [IndexModel([("bar", ASCENDING)])], session=session)) self._test_writes( lambda coll, session: coll.drop_index("foo_1", session=session)) self._test_writes( lambda coll, session: coll.drop_indexes(session=session)) self._test_writes( lambda coll, session: coll.reindex(session=session)) def _test_no_read_concern(self, op): coll = self.client.pymongo_test.test with self.client.start_session() as sess: coll.find_one({}, session=sess) operation_time = sess.operation_time self.assertIsNotNone(operation_time) self.listener.results.clear() op(coll, sess) rc = self.listener.results['started'][0].command.get( 'readConcern') self.assertIsNone(rc) @client_context.require_no_standalone def test_writes_do_not_include_read_concern(self): self._test_no_read_concern( lambda coll, session: coll.bulk_write( [InsertOne({})], session=session)) self._test_no_read_concern( lambda coll, session: coll.insert_one({}, session=session)) self._test_no_read_concern( lambda coll, session: coll.insert_many([{}], session=session)) self._test_no_read_concern( lambda coll, session: coll.replace_one( {'_id': 1}, {'x': 1}, session=session)) self._test_no_read_concern( lambda coll, session: coll.update_one( {}, {'$set': {'X': 1}}, session=session)) self._test_no_read_concern( lambda coll, session: coll.update_many( {}, {'$set': {'x': 1}}, session=session)) self._test_no_read_concern( lambda coll, session: coll.delete_one({}, session=session)) self._test_no_read_concern( lambda coll, session: coll.delete_many({}, session=session)) self._test_no_read_concern( lambda coll, session: coll.find_one_and_replace( {'x': 1}, {'y': 1}, session=session)) self._test_no_read_concern( lambda coll, session: coll.find_one_and_update( {'y': 1}, {'$set': {'x': 1}}, session=session)) self._test_no_read_concern( lambda coll, session: coll.find_one_and_delete( {'x': 1}, session=session)) self._test_no_read_concern( lambda coll, session: coll.create_index("foo", session=session)) self._test_no_read_concern( lambda coll, session: coll.create_indexes( [IndexModel([("bar", ASCENDING)])], session=session)) self._test_no_read_concern( lambda coll, session: coll.drop_index("foo_1", session=session)) self._test_no_read_concern( lambda coll, session: coll.drop_indexes(session=session)) self._test_no_read_concern( lambda coll, session: coll.reindex(session=session)) self._test_no_read_concern( lambda coll, session: list( coll.aggregate([{"$out": "aggout"}], session=session))) self._test_no_read_concern( lambda coll, session: coll.map_reduce( 'function() {}', 'function() {}', 'mrout', session=session)) # They are not writes, but currentOp and explain also don't support # readConcern. self._test_no_read_concern( lambda coll, session: coll.database.current_op(session=session)) self._test_no_read_concern( lambda coll, session: coll.find({}, session=session).explain()) @client_context.require_no_standalone def test_get_more_does_not_include_read_concern(self): coll = self.client.pymongo_test.test with self.client.start_session() as sess: coll.find_one({}, session=sess) operation_time = sess.operation_time self.assertIsNotNone(operation_time) coll.insert_many([{}, {}]) cursor = coll.find({}).batch_size(1) next(cursor) self.listener.results.clear() list(cursor) started = self.listener.results['started'][0] self.assertEqual(started.command_name, 'getMore') self.assertIsNone(started.command.get('readConcern')) def test_session_not_causal(self): with self.client.start_session(causal_consistency=False) as s: self.client.pymongo_test.test.insert_one({}, session=s) self.listener.results.clear() self.client.pymongo_test.test.find_one({}, session=s) act = self.listener.results['started'][0].command.get( 'readConcern', {}).get('afterClusterTime') self.assertIsNone(act) @client_context.require_standalone def test_server_not_causal(self): with self.client.start_session(causal_consistency=True) as s: self.client.pymongo_test.test.insert_one({}, session=s) self.listener.results.clear() self.client.pymongo_test.test.find_one({}, session=s) act = self.listener.results['started'][0].command.get( 'readConcern', {}).get('afterClusterTime') self.assertIsNone(act) @client_context.require_no_standalone def test_read_concern(self): with self.client.start_session(causal_consistency=True) as s: coll = self.client.pymongo_test.test coll.insert_one({}, session=s) self.listener.results.clear() coll.find_one({}, session=s) read_concern = self.listener.results['started'][0].command.get( 'readConcern') self.assertIsNotNone(read_concern) self.assertIsNone(read_concern.get('level')) self.assertIsNotNone(read_concern.get('afterClusterTime')) coll = coll.with_options(read_concern=ReadConcern("majority")) self.listener.results.clear() coll.find_one({}, session=s) read_concern = self.listener.results['started'][0].command.get( 'readConcern') self.assertIsNotNone(read_concern) self.assertEqual(read_concern.get('level'), 'majority') self.assertIsNotNone(read_concern.get('afterClusterTime')) def test_unacknowledged(self): with self.client.start_session(causal_consistency=True) as s: coll = self.client.pymongo_test.get_collection( 'test', write_concern=WriteConcern(w=0)) coll.insert_one({}, session=s) self.assertIsNone(s.operation_time) @client_context.require_no_standalone def test_cluster_time_with_server_support(self): self.client.pymongo_test.test.insert_one({}) self.listener.results.clear() self.client.pymongo_test.test.find_one({}) after_cluster_time = self.listener.results['started'][0].command.get( '$clusterTime') self.assertIsNotNone(after_cluster_time) @client_context.require_standalone def test_cluster_time_no_server_support(self): self.client.pymongo_test.test.insert_one({}) self.listener.results.clear() self.client.pymongo_test.test.find_one({}) after_cluster_time = self.listener.results['started'][0].command.get( '$clusterTime') self.assertIsNone(after_cluster_time) class TestSessionsMultiAuth(IntegrationTest): @client_context.require_auth @client_context.require_sessions def setUp(self): super(TestSessionsMultiAuth, self).setUp() client_context.create_user( 'pymongo_test', 'second-user', 'pass', roles=['readWrite']) self.addCleanup(client_context.drop_user, 'pymongo_test','second-user') @ignore_deprecations def test_session_authenticate_multiple(self): listener = SessionTestListener() # Logged in as root. client = rs_or_single_client(event_listeners=[listener]) db = client.pymongo_test db.authenticate('second-user', 'pass') with self.assertRaises(InvalidOperation): client.start_session() # No implicit sessions. listener.results.clear() db.collection.find_one() event = listener.first_command_started() self.assertNotIn( 'lsid', event.command, "find_one with multi-auth shouldn't have sent lsid with %s" % ( event.command_name)) @ignore_deprecations def test_explicit_session_logout(self): listener = SessionTestListener() # Changing auth invalidates the session. Start as root. client = rs_or_single_client(event_listeners=[listener]) db = client.pymongo_test db.collection.insert_many([{} for _ in range(10)]) self.addCleanup(db.collection.drop) with client.start_session() as s: listener.results.clear() cursor = db.collection.find(session=s).batch_size(2) next(cursor) event = listener.first_command_started() self.assertEqual(event.command_name, 'find') self.assertEqual( s.session_id, event.command.get('lsid'), "find() sent wrong lsid with %s cmd" % (event.command_name,)) client.admin.logout() db.authenticate('second-user', 'pass') err = ('Cannot use session after authenticating with different' ' credentials') with self.assertRaisesRegex(InvalidOperation, err): # Auth has changed between find and getMore. list(cursor) with self.assertRaisesRegex(InvalidOperation, err): db.collection.bulk_write([InsertOne({})], session=s) with self.assertRaisesRegex(InvalidOperation, err): db.collection_names(session=s) with self.assertRaisesRegex(InvalidOperation, err): db.collection.find_one(session=s) with self.assertRaisesRegex(InvalidOperation, err): list(db.collection.aggregate([], session=s)) @ignore_deprecations def test_implicit_session_logout(self): listener = SessionTestListener() # Changing auth doesn't invalidate the session. Start as root. client = rs_or_single_client(event_listeners=[listener]) db = client.pymongo_test for name, f in [ ('bulk_write', lambda: db.collection.bulk_write([InsertOne({})])), ('collection_names', db.collection_names), ('find_one', db.collection.find_one), ('aggregate', lambda: list(db.collection.aggregate([]))) ]: def sub_test(): listener.results.clear() f() for event in listener.results['started']: self.assertIn( 'lsid', event.command, "%s sent no lsid with %s" % ( name, event.command_name)) # We switch auth without clearing the pool of session ids. The # server considers these to be new sessions since it's a new user. # The old sessions time out on the server after 30 minutes. client.admin.logout() db.authenticate('second-user', 'pass') sub_test() db.logout() client.admin.authenticate(db_user, db_pwd) sub_test() class TestSessionsNotSupported(IntegrationTest): @client_context.require_version_max(3, 5, 10) def test_sessions_not_supported(self): with self.assertRaisesRegex( ConfigurationError, "Sessions are not supported"): self.client.start_session() class TestClusterTime(IntegrationTest): def setUp(self): super(TestClusterTime, self).setUp() if '$clusterTime' not in client_context.ismaster: raise SkipTest('$clusterTime not supported') @ignore_deprecations def test_cluster_time(self): listener = SessionTestListener() # Prevent heartbeats from updating $clusterTime between operations. client = rs_or_single_client(event_listeners=[listener], heartbeatFrequencyMS=999999) collection = client.pymongo_test.collection # Prepare for tests of find() and aggregate(). collection.insert_many([{} for _ in range(10)]) self.addCleanup(collection.drop) self.addCleanup(client.pymongo_test.collection2.drop) def bulk_insert(ordered): if ordered: bulk = collection.initialize_ordered_bulk_op() else: bulk = collection.initialize_unordered_bulk_op() bulk.insert({}) bulk.execute() def rename_and_drop(): # Ensure collection exists. collection.insert_one({}) collection.rename('collection2') client.pymongo_test.collection2.drop() def insert_and_find(): cursor = collection.find().batch_size(1) for _ in range(10): # Advance the cluster time. collection.insert_one({}) next(cursor) cursor.close() def insert_and_aggregate(): cursor = collection.aggregate([], batchSize=1).batch_size(1) for _ in range(5): # Advance the cluster time. collection.insert_one({}) next(cursor) cursor.close() ops = [ # Tests from Driver Sessions Spec. ('ping', lambda: client.admin.command('ping')), ('aggregate', lambda: list(collection.aggregate([]))), ('find', lambda: list(collection.find())), ('insert_one', lambda: collection.insert_one({})), # Additional PyMongo tests. ('insert_and_find', insert_and_find), ('insert_and_aggregate', insert_and_aggregate), ('update_one', lambda: collection.update_one({}, {'$set': {'x': 1}})), ('update_many', lambda: collection.update_many({}, {'$set': {'x': 1}})), ('delete_one', lambda: collection.delete_one({})), ('delete_many', lambda: collection.delete_many({})), ('bulk_write', lambda: collection.bulk_write([InsertOne({})])), ('ordered bulk', lambda: bulk_insert(True)), ('unordered bulk', lambda: bulk_insert(False)), ('rename_and_drop', rename_and_drop), ] for name, f in ops: listener.results.clear() # Call f() twice, insert to advance clusterTime, call f() again. f() f() collection.insert_one({}) f() self.assertGreaterEqual(len(listener.results['started']), 1) for i, event in enumerate(listener.results['started']): self.assertTrue( '$clusterTime' in event.command, "%s sent no $clusterTime with %s" % ( f.__name__, event.command_name)) if i > 0: succeeded = listener.results['succeeded'][i - 1] self.assertTrue( '$clusterTime' in succeeded.reply, "%s received no $clusterTime with %s" % ( f.__name__, succeeded.command_name)) self.assertTrue( event.command['$clusterTime']['clusterTime'] >= succeeded.reply['$clusterTime']['clusterTime'], "%s sent wrong $clusterTime with %s" % ( f.__name__, event.command_name)) pymongo-3.6.1/test/test_pooling.py0000644000076600000240000003653513245621354017513 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test built in connection-pooling with threads.""" import gc import random import socket import sys import threading import time from pymongo import MongoClient from pymongo.errors import (AutoReconnect, ConnectionFailure, DuplicateKeyError, ExceededMaxWaiters) sys.path[0:0] = [""] from pymongo.network import SocketChecker from pymongo.pool import Pool, PoolOptions from test import client_context, unittest from test.utils import (get_pool, joinall, delay, rs_or_single_client) @client_context.require_connection def setUpModule(): pass N = 10 DB = "pymongo-pooling-tests" def gc_collect_until_done(threads, timeout=60): start = time.time() running = list(threads) while running: assert (time.time() - start) < timeout, "Threads timed out" for t in running: t.join(0.1) if not t.isAlive(): running.remove(t) gc.collect() class MongoThread(threading.Thread): """A thread that uses a MongoClient.""" def __init__(self, client): super(MongoThread, self).__init__() self.daemon = True # Don't hang whole test if thread hangs. self.client = client self.db = self.client[DB] self.passed = False def run(self): self.run_mongo_thread() self.passed = True def run_mongo_thread(self): raise NotImplementedError class InsertOneAndFind(MongoThread): def run_mongo_thread(self): for _ in range(N): rand = random.randint(0, N) _id = self.db.sf.insert_one({"x": rand}).inserted_id assert rand == self.db.sf.find_one(_id)["x"] class Unique(MongoThread): def run_mongo_thread(self): for _ in range(N): self.db.unique.insert_one({}) # no error class NonUnique(MongoThread): def run_mongo_thread(self): for _ in range(N): try: self.db.unique.insert_one({"_id": "jesse"}) except DuplicateKeyError: pass else: raise AssertionError("Should have raised DuplicateKeyError") class Disconnect(MongoThread): def run_mongo_thread(self): for _ in range(N): self.client.close() class SocketGetter(MongoThread): """Utility for TestPooling. Checks out a socket and holds it forever. Used in test_no_wait_queue_timeout, test_wait_queue_multiple, and test_no_wait_queue_multiple. """ def __init__(self, client, pool): super(SocketGetter, self).__init__(client) self.state = 'init' self.pool = pool self.sock = None def run_mongo_thread(self): self.state = 'get_socket' # Pass 'checkout' so we can hold the socket. with self.pool.get_socket({}, checkout=True) as sock: self.sock = sock self.state = 'sock' def __del__(self): if self.sock: self.sock.close() def run_cases(client, cases): threads = [] n_runs = 5 for case in cases: for i in range(n_runs): t = case(client) t.start() threads.append(t) for t in threads: t.join() for t in threads: assert t.passed, "%s.run() threw an exception" % repr(t) class _TestPoolingBase(unittest.TestCase): """Base class for all connection-pool tests.""" def setUp(self): self.c = rs_or_single_client() db = self.c[DB] db.unique.drop() db.test.drop() db.unique.insert_one({"_id": "jesse"}) db.test.insert_many([{} for _ in range(10)]) def create_pool( self, pair=(client_context.host, client_context.port), *args, **kwargs): # Start the pool with the correct ssl options. pool_options = client_context.client._topology_settings.pool_options kwargs['ssl_context'] = pool_options.ssl_context kwargs['ssl_match_hostname'] = pool_options.ssl_match_hostname return Pool(pair, PoolOptions(*args, **kwargs)) class TestPooling(_TestPoolingBase): def test_max_pool_size_validation(self): host, port = client_context.host, client_context.port self.assertRaises( ValueError, MongoClient, host=host, port=port, maxPoolSize=-1) self.assertRaises( ValueError, MongoClient, host=host, port=port, maxPoolSize='foo') c = MongoClient(host=host, port=port, maxPoolSize=100, connect=False) self.assertEqual(c.max_pool_size, 100) def test_no_disconnect(self): run_cases(self.c, [NonUnique, Unique, InsertOneAndFind]) def test_disconnect(self): run_cases(self.c, [InsertOneAndFind, Disconnect, Unique]) def test_pool_reuses_open_socket(self): # Test Pool's _check_closed() method doesn't close a healthy socket. cx_pool = self.create_pool(max_pool_size=10) cx_pool._check_interval_seconds = 0 # Always check. with cx_pool.get_socket({}) as sock_info: pass with cx_pool.get_socket({}) as new_sock_info: self.assertEqual(sock_info, new_sock_info) self.assertEqual(1, len(cx_pool.sockets)) def test_get_socket_and_exception(self): # get_socket() returns socket after a non-network error. cx_pool = self.create_pool(max_pool_size=1, wait_queue_timeout=1) with self.assertRaises(ZeroDivisionError): with cx_pool.get_socket({}) as sock_info: 1 / 0 # Socket was returned, not closed. with cx_pool.get_socket({}) as new_sock_info: self.assertEqual(sock_info, new_sock_info) self.assertEqual(1, len(cx_pool.sockets)) def test_pool_removes_closed_socket(self): # Test that Pool removes explicitly closed socket. cx_pool = self.create_pool() with cx_pool.get_socket({}) as sock_info: # Use SocketInfo's API to close the socket. sock_info.close() self.assertEqual(0, len(cx_pool.sockets)) def test_pool_removes_dead_socket(self): # Test that Pool removes dead socket and the socket doesn't return # itself PYTHON-344 cx_pool = self.create_pool(max_pool_size=1, wait_queue_timeout=1) cx_pool._check_interval_seconds = 0 # Always check. with cx_pool.get_socket({}) as sock_info: # Simulate a closed socket without telling the SocketInfo it's # closed. sock_info.sock.close() self.assertTrue( cx_pool.socket_checker.socket_closed(sock_info.sock)) with cx_pool.get_socket({}) as new_sock_info: self.assertEqual(0, len(cx_pool.sockets)) self.assertNotEqual(sock_info, new_sock_info) self.assertEqual(1, len(cx_pool.sockets)) # Semaphore was released. with cx_pool.get_socket({}): pass def test_socket_closed(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((client_context.host, client_context.port)) socket_checker = SocketChecker() self.assertFalse(socket_checker.socket_closed(s)) s.close() self.assertTrue(socket_checker.socket_closed(s)) def test_socket_closed_thread_safe(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((client_context.host, client_context.port)) socket_checker = SocketChecker() def check_socket(): for _ in range(1000): self.assertFalse(socket_checker.socket_closed(s)) threads = [] for i in range(3): thread = threading.Thread(target=check_socket) thread.start() threads.append(thread) for thread in threads: thread.join() def test_return_socket_after_reset(self): pool = self.create_pool() with pool.get_socket({}) as sock: pool.reset() self.assertTrue(sock.closed) self.assertEqual(0, len(pool.sockets)) def test_pool_check(self): # Test that Pool recovers from two connection failures in a row. # This exercises code at the end of Pool._check(). cx_pool = self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) cx_pool._check_interval_seconds = 0 # Always check. with cx_pool.get_socket({}) as sock_info: # Simulate a closed socket without telling the SocketInfo it's # closed. sock_info.sock.close() # Swap pool's address with a bad one. address, cx_pool.address = cx_pool.address, ('foo.com', 1234) with self.assertRaises(AutoReconnect): with cx_pool.get_socket({}): pass # Back to normal, semaphore was correctly released. cx_pool.address = address with cx_pool.get_socket({}, checkout=True) as sock_info: pass sock_info.close() def test_wait_queue_timeout(self): wait_queue_timeout = 2 # Seconds pool = self.create_pool( max_pool_size=1, wait_queue_timeout=wait_queue_timeout) with pool.get_socket({}) as sock_info: start = time.time() with self.assertRaises(ConnectionFailure): with pool.get_socket({}): pass duration = time.time() - start self.assertTrue( abs(wait_queue_timeout - duration) < 1, "Waited %.2f seconds for a socket, expected %f" % ( duration, wait_queue_timeout)) sock_info.close() def test_no_wait_queue_timeout(self): # Verify get_socket() with no wait_queue_timeout blocks forever. pool = self.create_pool(max_pool_size=1) # Reach max_size. with pool.get_socket({}) as s1: t = SocketGetter(self.c, pool) t.start() while t.state != 'get_socket': time.sleep(0.1) time.sleep(1) self.assertEqual(t.state, 'get_socket') while t.state != 'sock': time.sleep(0.1) self.assertEqual(t.state, 'sock') self.assertEqual(t.sock, s1) s1.close() def test_wait_queue_multiple(self): wait_queue_multiple = 3 pool = self.create_pool( max_pool_size=2, wait_queue_multiple=wait_queue_multiple) # Reach max_size sockets. with pool.get_socket({}): with pool.get_socket({}): # Reach max_size * wait_queue_multiple waiters. threads = [] for _ in range(6): t = SocketGetter(self.c, pool) t.start() threads.append(t) time.sleep(1) for t in threads: self.assertEqual(t.state, 'get_socket') with self.assertRaises(ExceededMaxWaiters): with pool.get_socket({}): pass def test_no_wait_queue_multiple(self): pool = self.create_pool(max_pool_size=2) socks = [] for _ in range(2): # Pass 'checkout' so we can hold the socket. with pool.get_socket({}, checkout=True) as sock: socks.append(sock) threads = [] for _ in range(30): t = SocketGetter(self.c, pool) t.start() threads.append(t) time.sleep(1) for t in threads: self.assertEqual(t.state, 'get_socket') for socket_info in socks: socket_info.close() class TestPoolMaxSize(_TestPoolingBase): def test_max_pool_size(self): max_pool_size = 4 c = rs_or_single_client(maxPoolSize=max_pool_size) collection = c[DB].test # Need one document. collection.drop() collection.insert_one({}) # nthreads had better be much larger than max_pool_size to ensure that # max_pool_size sockets are actually required at some point in this # test's execution. cx_pool = get_pool(c) nthreads = 10 threads = [] lock = threading.Lock() self.n_passed = 0 def f(): for _ in range(5): collection.find_one({'$where': delay(0.1)}) assert len(cx_pool.sockets) <= max_pool_size with lock: self.n_passed += 1 for i in range(nthreads): t = threading.Thread(target=f) threads.append(t) t.start() joinall(threads) self.assertEqual(nthreads, self.n_passed) self.assertTrue(len(cx_pool.sockets) > 1) self.assertEqual(max_pool_size, cx_pool._socket_semaphore.counter) def test_max_pool_size_none(self): c = rs_or_single_client(maxPoolSize=None) collection = c[DB].test # Need one document. collection.drop() collection.insert_one({}) cx_pool = get_pool(c) nthreads = 10 threads = [] lock = threading.Lock() self.n_passed = 0 def f(): for _ in range(5): collection.find_one({'$where': delay(0.1)}) with lock: self.n_passed += 1 for i in range(nthreads): t = threading.Thread(target=f) threads.append(t) t.start() joinall(threads) self.assertEqual(nthreads, self.n_passed) self.assertTrue(len(cx_pool.sockets) > 1) def test_max_pool_size_zero(self): with self.assertRaises(ValueError): rs_or_single_client(maxPoolSize=0) def test_max_pool_size_with_connection_failure(self): # The pool acquires its semaphore before attempting to connect; ensure # it releases the semaphore on connection failure. test_pool = Pool( ('somedomainthatdoesntexist.org', 27017), PoolOptions( max_pool_size=1, connect_timeout=1, socket_timeout=1, wait_queue_timeout=1)) # First call to get_socket fails; if pool doesn't release its semaphore # then the second call raises "ConnectionFailure: Timed out waiting for # socket from pool" instead of AutoReconnect. for i in range(2): with self.assertRaises(AutoReconnect) as context: with test_pool.get_socket({}, checkout=True): pass # Testing for AutoReconnect instead of ConnectionFailure, above, # is sufficient right *now* to catch a semaphore leak. But that # seems error-prone, so check the message too. self.assertNotIn('waiting for socket from pool', str(context.exception)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_gridfs.py0000644000076600000240000004522713245621354017320 0ustar shanestaff00000000000000# -*- coding: utf-8 -*- # # Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the gridfs package. """ import sys sys.path[0:0] = [""] import datetime import threading import time import gridfs from bson.binary import Binary from bson.py3compat import StringIO, string_type from pymongo.mongo_client import MongoClient from pymongo.errors import (ConfigurationError, ConnectionFailure, ServerSelectionTimeoutError) from pymongo.read_preferences import ReadPreference from gridfs.errors import CorruptGridFile, FileExists, NoFile from test.test_replica_set_client import TestReplicaSetClientBase from test import (client_context, unittest, IntegrationTest) from test.utils import (joinall, single_client, one, rs_client, rs_or_single_client) class JustWrite(threading.Thread): def __init__(self, fs, n): threading.Thread.__init__(self) self.fs = fs self.n = n self.setDaemon(True) def run(self): for _ in range(self.n): file = self.fs.new_file(filename="test") file.write(b"hello") file.close() class JustRead(threading.Thread): def __init__(self, fs, n, results): threading.Thread.__init__(self) self.fs = fs self.n = n self.results = results self.setDaemon(True) def run(self): for _ in range(self.n): file = self.fs.get("test") data = file.read() self.results.append(data) assert data == b"hello" class TestGridfsNoConnect(unittest.TestCase): @classmethod def setUpClass(cls): cls.db = MongoClient(connect=False).pymongo_test def test_gridfs(self): self.assertRaises(TypeError, gridfs.GridFS, "foo") self.assertRaises(TypeError, gridfs.GridFS, self.db, 5) class TestGridfs(IntegrationTest): @classmethod def setUpClass(cls): super(TestGridfs, cls).setUpClass() cls.fs = gridfs.GridFS(cls.db) cls.alt = gridfs.GridFS(cls.db, "alt") def setUp(self): self.db.drop_collection("fs.files") self.db.drop_collection("fs.chunks") self.db.drop_collection("alt.files") self.db.drop_collection("alt.chunks") def test_basic(self): oid = self.fs.put(b"hello world") self.assertEqual(b"hello world", self.fs.get(oid).read()) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(1, self.db.fs.chunks.count()) self.fs.delete(oid) self.assertRaises(NoFile, self.fs.get, oid) self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) self.assertRaises(NoFile, self.fs.get, "foo") oid = self.fs.put(b"hello world", _id="foo") self.assertEqual("foo", oid) self.assertEqual(b"hello world", self.fs.get("foo").read()) def test_multi_chunk_delete(self): self.db.fs.drop() self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) gfs = gridfs.GridFS(self.db) oid = gfs.put(b"hello", chunkSize=1) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(5, self.db.fs.chunks.count()) gfs.delete(oid) self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) def test_list(self): self.assertEqual([], self.fs.list()) self.fs.put(b"hello world") self.assertEqual([], self.fs.list()) # PYTHON-598: in server versions before 2.5.x, creating an index on # filename, uploadDate causes list() to include None. self.fs.get_last_version() self.assertEqual([], self.fs.list()) self.fs.put(b"", filename="mike") self.fs.put(b"foo", filename="test") self.fs.put(b"", filename="hello world") self.assertEqual(set(["mike", "test", "hello world"]), set(self.fs.list())) def test_empty_file(self): oid = self.fs.put(b"") self.assertEqual(b"", self.fs.get(oid).read()) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) raw = self.db.fs.files.find_one() self.assertEqual(0, raw["length"]) self.assertEqual(oid, raw["_id"]) self.assertTrue(isinstance(raw["uploadDate"], datetime.datetime)) self.assertEqual(255 * 1024, raw["chunkSize"]) self.assertTrue(isinstance(raw["md5"], string_type)) def test_corrupt_chunk(self): files_id = self.fs.put(b'foobar') self.db.fs.chunks.update_one({'files_id': files_id}, {'$set': {'data': Binary(b'foo', 0)}}) try: out = self.fs.get(files_id) self.assertRaises(CorruptGridFile, out.read) out = self.fs.get(files_id) self.assertRaises(CorruptGridFile, out.readline) finally: self.fs.delete(files_id) def test_put_ensures_index(self): # setUp has dropped collections. names = self.db.collection_names() self.assertFalse([name for name in names if name.startswith('fs')]) chunks = self.db.fs.chunks files = self.db.fs.files self.fs.put(b"junk") self.assertTrue(any( info.get('key') == [('files_id', 1), ('n', 1)] for info in chunks.index_information().values())) self.assertTrue(any( info.get('key') == [('filename', 1), ('uploadDate', 1)] for info in files.index_information().values())) def test_alt_collection(self): oid = self.alt.put(b"hello world") self.assertEqual(b"hello world", self.alt.get(oid).read()) self.assertEqual(1, self.db.alt.files.count()) self.assertEqual(1, self.db.alt.chunks.count()) self.alt.delete(oid) self.assertRaises(NoFile, self.alt.get, oid) self.assertEqual(0, self.db.alt.files.count()) self.assertEqual(0, self.db.alt.chunks.count()) self.assertRaises(NoFile, self.alt.get, "foo") oid = self.alt.put(b"hello world", _id="foo") self.assertEqual("foo", oid) self.assertEqual(b"hello world", self.alt.get("foo").read()) self.alt.put(b"", filename="mike") self.alt.put(b"foo", filename="test") self.alt.put(b"", filename="hello world") self.assertEqual(set(["mike", "test", "hello world"]), set(self.alt.list())) def test_threaded_reads(self): self.fs.put(b"hello", _id="test") threads = [] results = [] for i in range(10): threads.append(JustRead(self.fs, 10, results)) threads[i].start() joinall(threads) self.assertEqual( 100 * [b'hello'], results ) def test_threaded_writes(self): threads = [] for i in range(10): threads.append(JustWrite(self.fs, 10)) threads[i].start() joinall(threads) f = self.fs.get_last_version("test") self.assertEqual(f.read(), b"hello") # Should have created 100 versions of 'test' file self.assertEqual( 100, self.db.fs.files.find({'filename': 'test'}).count() ) def test_get_last_version(self): one = self.fs.put(b"foo", filename="test") time.sleep(0.01) two = self.fs.new_file(filename="test") two.write(b"bar") two.close() time.sleep(0.01) two = two._id three = self.fs.put(b"baz", filename="test") self.assertEqual(b"baz", self.fs.get_last_version("test").read()) self.fs.delete(three) self.assertEqual(b"bar", self.fs.get_last_version("test").read()) self.fs.delete(two) self.assertEqual(b"foo", self.fs.get_last_version("test").read()) self.fs.delete(one) self.assertRaises(NoFile, self.fs.get_last_version, "test") def test_get_last_version_with_metadata(self): one = self.fs.put(b"foo", filename="test", author="author") time.sleep(0.01) two = self.fs.put(b"bar", filename="test", author="author") self.assertEqual(b"bar", self.fs.get_last_version(author="author").read()) self.fs.delete(two) self.assertEqual(b"foo", self.fs.get_last_version(author="author").read()) self.fs.delete(one) one = self.fs.put(b"foo", filename="test", author="author1") time.sleep(0.01) two = self.fs.put(b"bar", filename="test", author="author2") self.assertEqual(b"foo", self.fs.get_last_version(author="author1").read()) self.assertEqual(b"bar", self.fs.get_last_version(author="author2").read()) self.assertEqual(b"bar", self.fs.get_last_version(filename="test").read()) self.assertRaises(NoFile, self.fs.get_last_version, author="author3") self.assertRaises(NoFile, self.fs.get_last_version, filename="nottest", author="author1") self.fs.delete(one) self.fs.delete(two) def test_get_version(self): self.fs.put(b"foo", filename="test") time.sleep(0.01) self.fs.put(b"bar", filename="test") time.sleep(0.01) self.fs.put(b"baz", filename="test") time.sleep(0.01) self.assertEqual(b"foo", self.fs.get_version("test", 0).read()) self.assertEqual(b"bar", self.fs.get_version("test", 1).read()) self.assertEqual(b"baz", self.fs.get_version("test", 2).read()) self.assertEqual(b"baz", self.fs.get_version("test", -1).read()) self.assertEqual(b"bar", self.fs.get_version("test", -2).read()) self.assertEqual(b"foo", self.fs.get_version("test", -3).read()) self.assertRaises(NoFile, self.fs.get_version, "test", 3) self.assertRaises(NoFile, self.fs.get_version, "test", -4) def test_get_version_with_metadata(self): one = self.fs.put(b"foo", filename="test", author="author1") time.sleep(0.01) two = self.fs.put(b"bar", filename="test", author="author1") time.sleep(0.01) three = self.fs.put(b"baz", filename="test", author="author2") self.assertEqual(b"foo", self.fs.get_version(filename="test", author="author1", version=-2).read()) self.assertEqual(b"bar", self.fs.get_version(filename="test", author="author1", version=-1).read()) self.assertEqual(b"foo", self.fs.get_version(filename="test", author="author1", version=0).read()) self.assertEqual(b"bar", self.fs.get_version(filename="test", author="author1", version=1).read()) self.assertEqual(b"baz", self.fs.get_version(filename="test", author="author2", version=0).read()) self.assertEqual(b"baz", self.fs.get_version(filename="test", version=-1).read()) self.assertEqual(b"baz", self.fs.get_version(filename="test", version=2).read()) self.assertRaises(NoFile, self.fs.get_version, filename="test", author="author3") self.assertRaises(NoFile, self.fs.get_version, filename="test", author="author1", version=2) self.fs.delete(one) self.fs.delete(two) self.fs.delete(three) def test_put_filelike(self): oid = self.fs.put(StringIO(b"hello world"), chunk_size=1) self.assertEqual(11, self.db.fs.chunks.count()) self.assertEqual(b"hello world", self.fs.get(oid).read()) def test_file_exists(self): oid = self.fs.put(b"hello") self.assertRaises(FileExists, self.fs.put, b"world", _id=oid) one = self.fs.new_file(_id=123) one.write(b"some content") one.close() two = self.fs.new_file(_id=123) self.assertRaises(FileExists, two.write, b'x' * 262146) def test_exists(self): oid = self.fs.put(b"hello") self.assertTrue(self.fs.exists(oid)) self.assertTrue(self.fs.exists({"_id": oid})) self.assertTrue(self.fs.exists(_id=oid)) self.assertFalse(self.fs.exists(filename="mike")) self.assertFalse(self.fs.exists("mike")) oid = self.fs.put(b"hello", filename="mike", foo=12) self.assertTrue(self.fs.exists(oid)) self.assertTrue(self.fs.exists({"_id": oid})) self.assertTrue(self.fs.exists(_id=oid)) self.assertTrue(self.fs.exists(filename="mike")) self.assertTrue(self.fs.exists({"filename": "mike"})) self.assertTrue(self.fs.exists(foo=12)) self.assertTrue(self.fs.exists({"foo": 12})) self.assertTrue(self.fs.exists(foo={"$gt": 11})) self.assertTrue(self.fs.exists({"foo": {"$gt": 11}})) self.assertFalse(self.fs.exists(foo=13)) self.assertFalse(self.fs.exists({"foo": 13})) self.assertFalse(self.fs.exists(foo={"$gt": 12})) self.assertFalse(self.fs.exists({"foo": {"$gt": 12}})) def test_put_unicode(self): self.assertRaises(TypeError, self.fs.put, u"hello") oid = self.fs.put(u"hello", encoding="utf-8") self.assertEqual(b"hello", self.fs.get(oid).read()) self.assertEqual("utf-8", self.fs.get(oid).encoding) oid = self.fs.put(u"aé", encoding="iso-8859-1") self.assertEqual(u"aé".encode("iso-8859-1"), self.fs.get(oid).read()) self.assertEqual("iso-8859-1", self.fs.get(oid).encoding) def test_missing_length_iter(self): # Test fix that guards against PHP-237 self.fs.put(b"", filename="empty") doc = self.db.fs.files.find_one({"filename": "empty"}) doc.pop("length") self.db.fs.files.replace_one({"_id": doc["_id"]}, doc) f = self.fs.get_last_version(filename="empty") def iterate_file(grid_file): for chunk in grid_file: pass return True self.assertTrue(iterate_file(f)) def test_gridfs_lazy_connect(self): client = MongoClient('badhost', connect=False, serverSelectionTimeoutMS=10) db = client.db gfs = gridfs.GridFS(db) self.assertRaises(ServerSelectionTimeoutError, gfs.list) fs = gridfs.GridFS(db) f = fs.new_file() self.assertRaises(ServerSelectionTimeoutError, f.close) def test_gridfs_find(self): self.fs.put(b"test2", filename="two") time.sleep(0.01) self.fs.put(b"test2+", filename="two") time.sleep(0.01) self.fs.put(b"test1", filename="one") time.sleep(0.01) self.fs.put(b"test2++", filename="two") self.assertEqual(3, self.fs.find({"filename": "two"}).count()) self.assertEqual(4, self.fs.find().count()) cursor = self.fs.find( no_cursor_timeout=False).sort("uploadDate", -1).skip(1).limit(2) gout = next(cursor) self.assertEqual(b"test1", gout.read()) cursor.rewind() gout = next(cursor) self.assertEqual(b"test1", gout.read()) gout = next(cursor) self.assertEqual(b"test2+", gout.read()) self.assertRaises(StopIteration, cursor.__next__) cursor.close() self.assertRaises(TypeError, self.fs.find, {}, {"_id": True}) def test_gridfs_find_one(self): self.assertEqual(None, self.fs.find_one()) id1 = self.fs.put(b'test1', filename='file1') self.assertEqual(b'test1', self.fs.find_one().read()) id2 = self.fs.put(b'test2', filename='file2', meta='data') self.assertEqual(b'test1', self.fs.find_one(id1).read()) self.assertEqual(b'test2', self.fs.find_one(id2).read()) self.assertEqual(b'test1', self.fs.find_one({'filename': 'file1'}).read()) self.assertEqual('data', self.fs.find_one(id2).meta) def test_grid_in_non_int_chunksize(self): # Lua, and perhaps other buggy GridFS clients, store size as a float. data = b'data' self.fs.put(data, filename='f') self.db.fs.files.update_one({'filename': 'f'}, {'$set': {'chunkSize': 100.0}}) self.assertEqual(data, self.fs.get_version('f').read()) def test_unacknowledged(self): # w=0 is prohibited. with self.assertRaises(ConfigurationError): gridfs.GridFS(rs_or_single_client(w=0).pymongo_test) class TestGridfsReplicaSet(TestReplicaSetClientBase): @classmethod @client_context.require_secondaries_count(1) def setUpClass(cls): super(TestGridfsReplicaSet, cls).setUpClass() @classmethod def tearDownClass(cls): client_context.client.drop_database('gfsreplica') def test_gridfs_replica_set(self): rsc = rs_client( w=self.w, read_preference=ReadPreference.SECONDARY) fs = gridfs.GridFS(rsc.gfsreplica, 'gfsreplicatest') gin = fs.new_file() self.assertEqual(gin._coll.read_preference, ReadPreference.PRIMARY) oid = fs.put(b'foo') content = fs.get(oid).read() self.assertEqual(b'foo', content) def test_gridfs_secondary(self): primary_host, primary_port = self.primary primary_connection = single_client(primary_host, primary_port) secondary_host, secondary_port = one(self.secondaries) secondary_connection = single_client( secondary_host, secondary_port, read_preference=ReadPreference.SECONDARY) # Should detect it's connected to secondary and not attempt to # create index fs = gridfs.GridFS(secondary_connection.gfsreplica, 'gfssecondarytest') # This won't detect secondary, raises error self.assertRaises(ConnectionFailure, fs.put, b'foo') def test_gridfs_secondary_lazy(self): # Should detect it's connected to secondary and not attempt to # create index. secondary_host, secondary_port = one(self.secondaries) client = single_client( secondary_host, secondary_port, read_preference=ReadPreference.SECONDARY, connect=False) # Still no connection. fs = gridfs.GridFS(client.gfsreplica, 'gfssecondarylazytest') # Connects, doesn't create index. self.assertRaises(NoFile, fs.get_last_version) self.assertRaises(ConnectionFailure, fs.put, 'data') if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_retryable_writes.py0000644000076600000240000004507213245621354021426 0ustar shanestaff00000000000000# Copyright 2017 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test retryable writes.""" import json import os import sys sys.path[0:0] = [""] from bson.int64 import Int64 from bson.objectid import ObjectId from bson.son import SON from pymongo.errors import (ConnectionFailure, ServerSelectionTimeoutError) from pymongo.monitoring import _SENSITIVE_COMMANDS from pymongo.mongo_client import MongoClient from pymongo.operations import (InsertOne, DeleteMany, DeleteOne, ReplaceOne, UpdateMany, UpdateOne) from pymongo.write_concern import WriteConcern from test import unittest, client_context, IntegrationTest, SkipTest from test.utils import rs_or_single_client, EventListener, DeprecationFilter from test.test_crud import check_result, run_operation # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'retryable_writes') class CommandListener(EventListener): def started(self, event): if event.command_name.lower() in _SENSITIVE_COMMANDS: return super(CommandListener, self).started(event) def succeeded(self, event): if event.command_name.lower() in _SENSITIVE_COMMANDS: return super(CommandListener, self).succeeded(event) def failed(self, event): if event.command_name.lower() in _SENSITIVE_COMMANDS: return super(CommandListener, self).failed(event) class TestAllScenarios(IntegrationTest): @classmethod @client_context.require_version_min(3, 5) @client_context.require_replica_set @client_context.require_test_commands def setUpClass(cls): super(TestAllScenarios, cls).setUpClass() cls.client = rs_or_single_client(retryWrites=True) cls.db = cls.client.pymongo_test def tearDown(self): self.client.admin.command(SON([ ('configureFailPoint', 'onPrimaryTransactionalWrite'), ('mode', 'off')])) def set_fail_point(self, command_args): cmd = SON([('configureFailPoint', 'onPrimaryTransactionalWrite')]) cmd.update(command_args) self.client.admin.command(cmd) def create_test(scenario_def, test): def run_scenario(self): # Load data. assert scenario_def['data'], "tests must have non-empty data" self.db.test.drop() self.db.test.insert_many(scenario_def['data']) # Set the failPoint self.set_fail_point(test['failPoint']) test_outcome = test['outcome'] should_fail = test_outcome.get('error') result = None error = None try: result = run_operation(self.db.test, test) except ConnectionFailure as exc: error = exc if should_fail: self.assertIsNotNone(error, 'should have raised an error') else: self.assertIsNone(error, 'should not have raised an error') # Assert final state is expected. expected_c = test_outcome.get('collection') if expected_c is not None: expected_name = expected_c.get('name') if expected_name is not None: db_coll = self.db[expected_name] else: db_coll = self.db.test self.assertEqual(list(db_coll.find()), expected_c['data']) expected_result = test_outcome.get('result') # We can't test the expected result when the test should fail because # the BulkWriteResult is not reported when raising a network error. if not should_fail: self.assertTrue(check_result(expected_result, result), "%r != %r" % (expected_result, result)) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): dirname = os.path.split(dirpath)[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json.load(scenario_stream) test_type = os.path.splitext(filename)[0] # Construct test from scenario. for test in scenario_def['tests']: new_test = create_test(scenario_def, test) test_name = 'test_%s_%s_%s' % ( dirname, test_type, str(test['description'].replace(" ", "_"))) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() def retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {}), (coll.bulk_write, [[InsertOne({}), InsertOne({})]], {'ordered': False}), (coll.bulk_write, [[ReplaceOne({}, {})]], {}), (coll.bulk_write, [[ReplaceOne({}, {}), ReplaceOne({}, {})]], {}), (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), UpdateOne({}, {'$set': {'a': 1}})]], {}), (coll.bulk_write, [[DeleteOne({})]], {}), (coll.bulk_write, [[DeleteOne({}), DeleteOne({})]], {}), (coll.insert_one, [{}], {}), (coll.insert_many, [[{}, {}]], {}), (coll.replace_one, [{}, {}], {}), (coll.update_one, [{}, {'$set': {'a': 1}}], {}), (coll.delete_one, [{}], {}), (coll.find_one_and_replace, [{}, {'a': 3}], {}), (coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}), (coll.find_one_and_delete, [{}, {}], {}), # Deprecated methods. # Insert with single or multiple documents. (coll.insert, [{}], {}), (coll.insert, [[{}]], {}), (coll.insert, [[{}, {}]], {}), # Save with and without an _id. (coll.save, [{}], {}), (coll.save, [{'_id': ObjectId()}], {}), # Non-multi update. (coll.update, [{}, {'$set': {'a': 1}}], {}), # Non-multi remove. (coll.remove, [{}], {'multi': False}), # Replace. (coll.find_and_modify, [{}, {'a': 3}], {}), # Update. (coll.find_and_modify, [{}, {'$set': {'a': 1}}], {}), # Delete. (coll.find_and_modify, [{}, {}], {'remove': True}), ] def non_retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), UpdateMany({}, {'$set': {'a': 1}})]], {}), (coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_many, [{}], {}), # Deprecated methods. # Multi remove. (coll.remove, [{}], {}), # Multi update. (coll.update, [{}, {'$set': {'a': 1}}], {'multi': True}), # Unacknowledged deprecated methods. (coll.insert, [{}], {'w': 0}), # Unacknowledged Non-multi update. (coll.update, [{}, {'$set': {'a': 1}}], {'w': 0}), # Unacknowledged Non-multi remove. (coll.remove, [{}], {'multi': False, 'w': 0}), # Unacknowledged Replace. (coll.find_and_modify, [{}, {'a': 3}], {'writeConcern': {'w': 0}}), # Unacknowledged Update. (coll.find_and_modify, [{}, {'$set': {'a': 1}}], {'writeConcern': {'w': 0}}), # Unacknowledged Delete. (coll.find_and_modify, [{}, {}], {'remove': True, 'writeConcern': {'w': 0}}), ] class IgnoreDeprecationsTest(IntegrationTest): @classmethod def setUpClass(cls): super(IgnoreDeprecationsTest, cls).setUpClass() cls.deprecation_filter = DeprecationFilter() @classmethod def tearDownClass(cls): cls.deprecation_filter.stop() super(IgnoreDeprecationsTest, cls).tearDownClass() class TestRetryableWrites(IgnoreDeprecationsTest): @classmethod def setUpClass(cls): super(TestRetryableWrites, cls).setUpClass() cls.listener = CommandListener() cls.client = rs_or_single_client( retryWrites=True, event_listeners=[cls.listener]) cls.db = cls.client.pymongo_test def setUp(self): if (client_context.version.at_least(3, 5) and client_context.is_rs and client_context.test_commands_enabled): self.client.admin.command(SON([ ('configureFailPoint', 'onPrimaryTransactionalWrite'), ('mode', 'alwaysOn')])) def tearDown(self): if (client_context.version.at_least(3, 5) and client_context.is_rs and client_context.test_commands_enabled): self.client.admin.command(SON([ ('configureFailPoint', 'onPrimaryTransactionalWrite'), ('mode', 'off')])) def test_supported_single_statement_no_retry(self): listener = CommandListener() client = rs_or_single_client( retryWrites=False, event_listeners=[listener]) self.addCleanup(client.close) for method, args, kwargs in retryable_single_statement_ops( client.db.retryable_write_test): msg = '%s(*%r, **%r)' % (method.__name__, args, kwargs) listener.results.clear() method(*args, **kwargs) for event in listener.results['started']: self.assertNotIn( 'txnNumber', event.command, '%s sent txnNumber with %s' % (msg, event.command_name)) @client_context.require_version_min(3, 5) @client_context.require_no_standalone def test_supported_single_statement_supported_cluster(self): for method, args, kwargs in retryable_single_statement_ops( self.db.retryable_write_test): msg = '%s(*%r, **%r)' % (method.__name__, args, kwargs) self.listener.results.clear() method(*args, **kwargs) commands_started = self.listener.results['started'] self.assertEqual(len(self.listener.results['succeeded']), 1, msg) first_attempt = commands_started[0] self.assertIn( 'lsid', first_attempt.command, '%s sent no lsid with %s' % (msg, first_attempt.command_name)) initial_session_id = first_attempt.command['lsid'] self.assertIn( 'txnNumber', first_attempt.command, '%s sent no txnNumber with %s' % ( msg, first_attempt.command_name)) # There should be no retry when the failpoint is not active. if (client_context.is_mongos or not client_context.test_commands_enabled): self.assertEqual(len(commands_started), 1) continue initial_transaction_id = first_attempt.command['txnNumber'] retry_attempt = commands_started[1] self.assertIn( 'lsid', retry_attempt.command, '%s sent no lsid with %s' % (msg, first_attempt.command_name)) self.assertEqual( retry_attempt.command['lsid'], initial_session_id, msg) self.assertIn( 'txnNumber', retry_attempt.command, '%s sent no txnNumber with %s' % ( msg, first_attempt.command_name)) self.assertEqual(retry_attempt.command['txnNumber'], initial_transaction_id, msg) def test_supported_single_statement_unsupported_cluster(self): if client_context.version.at_least(3, 5) and ( client_context.is_rs or client_context.is_mongos): raise SkipTest('This cluster supports retryable writes') for method, args, kwargs in retryable_single_statement_ops( self.db.retryable_write_test): msg = '%s(*%r, **%r)' % (method.__name__, args, kwargs) self.listener.results.clear() method(*args, **kwargs) for event in self.listener.results['started']: self.assertNotIn( 'txnNumber', event.command, '%s sent txnNumber with %s' % (msg, event.command_name)) def test_unsupported_single_statement(self): coll = self.db.retryable_write_test coll.insert_many([{}, {}]) coll_w0 = coll.with_options(write_concern=WriteConcern(w=0)) for method, args, kwargs in (non_retryable_single_statement_ops(coll) + retryable_single_statement_ops(coll_w0)): msg = '%s(*%r, **%r)' % (method.__name__, args, kwargs) self.listener.results.clear() method(*args, **kwargs) started_events = self.listener.results['started'] self.assertEqual(len(self.listener.results['succeeded']), len(started_events), msg) self.assertEqual(len(self.listener.results['failed']), 0, msg) for event in started_events: self.assertNotIn( 'txnNumber', event.command, '%s sent txnNumber with %s' % (msg, event.command_name)) def test_server_selection_timeout_not_retried(self): """A ServerSelectionTimeoutError is not retried.""" listener = CommandListener() client = MongoClient( 'somedomainthatdoesntexist.org', serverSelectionTimeoutMS=1, retryWrites=True, event_listeners=[listener]) for method, args, kwargs in retryable_single_statement_ops( client.db.retryable_write_test): msg = '%s(*%r, **%r)' % (method.__name__, args, kwargs) listener.results.clear() with self.assertRaises(ServerSelectionTimeoutError, msg=msg): method(*args, **kwargs) self.assertEqual(len(listener.results['started']), 0, msg) @client_context.require_version_min(3, 5) @client_context.require_replica_set @client_context.require_test_commands def test_retry_timeout_raises_original_error(self): """A ServerSelectionTimeoutError on the retry attempt raises the original error. """ listener = CommandListener() client = rs_or_single_client( retryWrites=True, event_listeners=[listener]) self.addCleanup(client.close) topology = client._topology select_server = topology.select_server def mock_select_server(*args, **kwargs): server = select_server(*args, **kwargs) def raise_error(*args, **kwargs): raise ServerSelectionTimeoutError( 'No primary available for writes') # Raise ServerSelectionTimeout on the retry attempt. topology.select_server = raise_error return server for method, args, kwargs in retryable_single_statement_ops( client.db.retryable_write_test): msg = '%s(*%r, **%r)' % (method.__name__, args, kwargs) listener.results.clear() topology.select_server = mock_select_server with self.assertRaises(ConnectionFailure, msg=msg): method(*args, **kwargs) self.assertEqual(len(listener.results['started']), 1, msg) @client_context.require_version_min(3, 5) @client_context.require_replica_set @client_context.require_test_commands def test_batch_splitting(self): """Test retry succeeds after failures during batch splitting.""" large = 's' * 1024 * 1024 * 15 coll = self.db.retryable_write_test coll.delete_many({}) self.listener.results.clear() coll.bulk_write([ InsertOne({'_id': 1, 'l': large}), InsertOne({'_id': 2, 'l': large}), InsertOne({'_id': 3, 'l': large}), UpdateOne({'_id': 1, 'l': large}, {'$unset': {'l': 1}, '$inc': {'count': 1}}), UpdateOne({'_id': 2, 'l': large}, {'$set': {'foo': 'bar'}}), DeleteOne({'l': large}), DeleteOne({'l': large})]) # Each command should fail and be retried. self.assertEqual(len(self.listener.results['started']), 14) self.assertEqual(coll.find_one(), {'_id': 1, 'count': 1}) @client_context.require_version_min(3, 5) @client_context.require_replica_set @client_context.require_test_commands def test_batch_splitting_retry_fails(self): """Test retry fails during batch splitting.""" large = 's' * 1024 * 1024 * 15 coll = self.db.retryable_write_test coll.delete_many({}) self.client.admin.command(SON([ ('configureFailPoint', 'onPrimaryTransactionalWrite'), ('mode', {'skip': 1}), ('data', {'failBeforeCommitExceptionCode': 1})])) self.listener.results.clear() with self.client.start_session() as session: initial_txn = session._server_session._transaction_id try: coll.bulk_write([InsertOne({'_id': 1, 'l': large}), InsertOne({'_id': 2, 'l': large}), InsertOne({'_id': 3, 'l': large})], session=session) except ConnectionFailure: pass else: self.fail("bulk_write should have failed") started = self.listener.results['started'] self.assertEqual(len(started), 3) self.assertEqual(len(self.listener.results['succeeded']), 1) expected_txn = Int64(initial_txn + 1) self.assertEqual(started[0].command['txnNumber'], expected_txn) self.assertEqual(started[0].command['lsid'], session.session_id) expected_txn = Int64(initial_txn + 2) self.assertEqual(started[1].command['txnNumber'], expected_txn) self.assertEqual(started[1].command['lsid'], session.session_id) started[1].command.pop('$clusterTime') started[2].command.pop('$clusterTime') self.assertEqual(started[1].command, started[2].command) final_txn = session._server_session._transaction_id self.assertEqual(final_txn, expected_txn) self.assertEqual(coll.find_one(projection={'_id': True}), {'_id': 1}) if __name__ == '__main__': unittest.main() pymongo-3.6.1/test/test_pymongo.py0000644000076600000240000000174413245617773017540 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the pymongo module itself.""" import sys sys.path[0:0] = [""] import pymongo from test import unittest class TestPyMongo(unittest.TestCase): def test_mongo_client_alias(self): # Testing that pymongo module imports mongo_client.MongoClient self.assertEqual(pymongo.MongoClient, pymongo.mongo_client.MongoClient) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_code.py0000644000076600000240000000737613245621354016757 0ustar shanestaff00000000000000# -*- coding: utf-8 -*- # # Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the Code wrapper.""" import sys sys.path[0:0] = [""] from bson.code import Code from test import unittest class TestCode(unittest.TestCase): def test_types(self): self.assertRaises(TypeError, Code, 5) self.assertRaises(TypeError, Code, None) self.assertRaises(TypeError, Code, "aoeu", 5) self.assertRaises(TypeError, Code, u"aoeu", 5) self.assertTrue(Code("aoeu")) self.assertTrue(Code(u"aoeu")) self.assertTrue(Code("aoeu", {})) self.assertTrue(Code(u"aoeu", {})) def test_read_only(self): c = Code("blah") def set_c(): c.scope = 5 self.assertRaises(AttributeError, set_c) def test_code(self): a_string = "hello world" a_code = Code("hello world") self.assertTrue(a_code.startswith("hello")) self.assertTrue(a_code.endswith("world")) self.assertTrue(isinstance(a_code, Code)) self.assertFalse(isinstance(a_string, Code)) self.assertIsNone(a_code.scope) with_scope = Code('hello world', {'my_var': 5}) self.assertEqual({'my_var': 5}, with_scope.scope) empty_scope = Code('hello world', {}) self.assertEqual({}, empty_scope.scope) another_scope = Code(with_scope, {'new_var': 42}) self.assertEqual(str(with_scope), str(another_scope)) self.assertEqual({'new_var': 42, 'my_var': 5}, another_scope.scope) # No error. Code(u'héllø world¡') def test_repr(self): c = Code("hello world", {}) self.assertEqual(repr(c), "Code('hello world', {})") c.scope["foo"] = "bar" self.assertEqual(repr(c), "Code('hello world', {'foo': 'bar'})") c = Code("hello world", {"blah": 3}) self.assertEqual(repr(c), "Code('hello world', {'blah': 3})") c = Code("\x08\xFF") self.assertEqual(repr(c), "Code(%s, None)" % (repr("\x08\xFF"),)) def test_equality(self): b = Code("hello") c = Code("hello", {"foo": 5}) self.assertNotEqual(b, c) self.assertEqual(c, Code("hello", {"foo": 5})) self.assertNotEqual(c, Code("hello", {"foo": 6})) self.assertEqual(b, Code("hello")) self.assertEqual(b, Code("hello", None)) self.assertNotEqual(b, Code("hello ")) self.assertNotEqual("hello", Code("hello")) # Explicitly test inequality self.assertFalse(c != Code("hello", {"foo": 5})) self.assertFalse(b != Code("hello")) self.assertFalse(b != Code("hello", None)) def test_hash(self): self.assertRaises(TypeError, hash, Code("hello world")) def test_scope_preserved(self): a = Code("hello") b = Code("hello", {"foo": 5}) self.assertEqual(a, Code(a)) self.assertEqual(b, Code(b)) self.assertNotEqual(a, Code(b)) self.assertNotEqual(b, Code(a)) def test_scope_kwargs(self): self.assertEqual({"a": 1}, Code("", a=1).scope) self.assertEqual({"a": 1}, Code("", {"a": 2}, a=1).scope) self.assertEqual({"a": 1, "b": 2, "c": 3}, Code("", {"b": 2}, a=1, c=3).scope) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_bson_corpus.py0000644000076600000240000002272413245621354020373 0ustar shanestaff00000000000000# Copyright 2016-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Run the BSON corpus specification tests.""" import binascii import codecs import functools import glob import os import sys from decimal import DecimalException if sys.version_info[:2] == (2, 6): try: import simplejson as json except ImportError: import json else: import json sys.path[0:0] = [""] from bson import BSON, json_util from bson.binary import STANDARD from bson.codec_options import CodecOptions from bson.decimal128 import Decimal128 from bson.dbref import DBRef from bson.errors import InvalidBSON, InvalidId from bson.json_util import JSONMode from bson.py3compat import text_type, b from bson.son import SON from test import unittest _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'bson_corpus') _TESTS_TO_SKIP = set([ # Python cannot decode dates after year 9999. 'Y10K', ]) _NON_PARSE_ERRORS = set([ # {"$date": } is our legacy format which we still need to parse. 'Bad $date (number, not string or hash)', # This variant of $numberLong may have been generated by an old version # of mongoexport. 'Bad $numberLong (number, not string)', ]) _DEPRECATED_BSON_TYPES = { # Symbol '0x0E': text_type, # Undefined '0x06': type(None), # DBPointer '0x0C': DBRef } # Need to set tz_aware=True in order to use "strict" dates in extended JSON. codec_options = CodecOptions(tz_aware=True, document_class=SON) # We normally encode UUID as binary subtype 0x03, # but we'll need to encode to subtype 0x04 for one of the tests. codec_options_uuid_04 = codec_options._replace(uuid_representation=STANDARD) json_options_uuid_04 = json_util.JSONOptions(json_mode=JSONMode.CANONICAL, uuid_representation=STANDARD) json_options_iso8601 = json_util.JSONOptions( datetime_representation=json_util.DatetimeRepresentation.ISO8601) to_extjson = functools.partial(json_util.dumps, json_options=json_util.CANONICAL_JSON_OPTIONS) to_extjson_uuid_04 = functools.partial(json_util.dumps, json_options=json_options_uuid_04) to_extjson_iso8601 = functools.partial(json_util.dumps, json_options=json_options_iso8601) to_relaxed_extjson = functools.partial( json_util.dumps, json_options=json_util.RELAXED_JSON_OPTIONS) to_bson_uuid_04 = functools.partial(BSON.encode, codec_options=codec_options_uuid_04) to_bson = functools.partial(BSON.encode, codec_options=codec_options) decode_bson = lambda bbytes: BSON(bbytes).decode(codec_options=codec_options) if json_util._HAS_OBJECT_PAIRS_HOOK: decode_extjson = functools.partial( json_util.loads, json_options=json_util.JSONOptions(json_mode=JSONMode.CANONICAL, document_class=SON)) loads = functools.partial(json.loads, object_pairs_hook=SON) else: decode_extjson = functools.partial( json_util.loads, json_options=json_util.CANONICAL_JSON_OPTIONS) loads = json.loads class TestBSONCorpus(unittest.TestCase): def assertJsonEqual(self, first, second, msg=None): """Fail if the two json strings are unequal. Normalize json by parsing it with the built-in json library. This accounts for discrepancies in spacing. """ self.assertEqual(loads(first), loads(second), msg=msg) def create_test(case_spec): bson_type = case_spec['bson_type'] # Test key is absent when testing top-level documents. test_key = case_spec.get('test_key') deprecated = case_spec.get('deprecated') def run_test(self): for valid_case in case_spec.get('valid', []): description = valid_case['description'] if description in _TESTS_TO_SKIP: continue # Special case for testing encoding UUID as binary subtype 0x04. if description == 'subtype 0x04': encode_extjson = to_extjson_uuid_04 encode_bson = to_bson_uuid_04 else: encode_extjson = to_extjson encode_bson = to_bson cB = binascii.unhexlify(b(valid_case['canonical_bson'])) cEJ = valid_case['canonical_extjson'] rEJ = valid_case.get('relaxed_extjson') dEJ = valid_case.get('degenerate_extjson') lossy = valid_case.get('lossy') decoded_bson = decode_bson(cB) if not lossy: # Make sure we can parse the legacy (default) JSON format. legacy_json = json_util.dumps( decoded_bson, json_options=json_util.LEGACY_JSON_OPTIONS) self.assertEqual(decode_extjson(legacy_json), decoded_bson) if deprecated: if 'converted_bson' in valid_case: converted_bson = binascii.unhexlify( b(valid_case['converted_bson'])) self.assertEqual(encode_bson(decoded_bson), converted_bson) self.assertJsonEqual( encode_extjson(decode_bson(converted_bson)), valid_case['converted_extjson']) # Make sure we can decode the type. self.assertEqual(decoded_bson, decode_extjson(cEJ)) if test_key is not None: self.assertIsInstance(decoded_bson[test_key], _DEPRECATED_BSON_TYPES[bson_type]) continue # PyPy3 and Jython can't handle NaN with a payload from # struct.(un)pack if endianness is specified in the format string. if not ((('PyPy' in sys.version and sys.version_info[:2] < (3, 3)) or sys.platform.startswith("java")) and description == 'NaN with payload'): # Test round-tripping canonical bson. self.assertEqual(encode_bson(decoded_bson), cB) self.assertJsonEqual(encode_extjson(decoded_bson), cEJ) # Test round-tripping canonical extended json. decoded_json = decode_extjson(cEJ) self.assertJsonEqual(encode_extjson(decoded_json), cEJ) if not lossy and json_util._HAS_OBJECT_PAIRS_HOOK: self.assertEqual(encode_bson(decoded_json), cB) # Test round-tripping degenerate bson. if 'degenerate_bson' in valid_case: dB = binascii.unhexlify(b(valid_case['degenerate_bson'])) self.assertEqual(encode_bson(decode_bson(dB)), cB) # Test round-tripping degenerate extended json. if dEJ is not None: decoded_json = decode_extjson(dEJ) self.assertJsonEqual(encode_extjson(decoded_json), cEJ) if not lossy: # We don't need to check json_util._HAS_OBJECT_PAIRS_HOOK # because degenerate_extjson is always a single key so # the order cannot be changed. self.assertEqual(encode_bson(decoded_json), cB) # Test round-tripping relaxed extended json. if rEJ is not None: self.assertJsonEqual(to_relaxed_extjson(decoded_bson), rEJ) decoded_json = decode_extjson(rEJ) self.assertJsonEqual(to_relaxed_extjson(decoded_json), rEJ) for decode_error_case in case_spec.get('decodeErrors', []): with self.assertRaises(InvalidBSON): decode_bson( binascii.unhexlify(b(decode_error_case['bson']))) for parse_error_case in case_spec.get('parseErrors', []): if bson_type == '0x13': self.assertRaises( DecimalException, Decimal128, parse_error_case['string']) elif bson_type == '0x00': description = parse_error_case['description'] if description in _NON_PARSE_ERRORS: decode_extjson(parse_error_case['string']) else: try: decode_extjson(parse_error_case['string']) raise AssertionError('exception not raised for test ' 'case: ' + description) except (ValueError, KeyError, TypeError, InvalidId): pass else: raise AssertionError('cannot test parseErrors for type ' + bson_type) return run_test def create_tests(): for filename in glob.glob(os.path.join(_TEST_PATH, '*.json')): test_suffix, _ = os.path.splitext(os.path.basename(filename)) with codecs.open(filename, encoding='utf-8') as bson_test_file: test_method = create_test(json.load(bson_test_file)) setattr(TestBSONCorpus, 'test_' + test_suffix, test_method) create_tests() if __name__ == '__main__': unittest.main() pymongo-3.6.1/test/test_timestamp.py0000644000076600000240000000527113245617773020052 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the Timestamp class.""" import datetime import sys import copy import pickle sys.path[0:0] = [""] from bson.timestamp import Timestamp from bson.tz_util import utc from test import unittest class TestTimestamp(unittest.TestCase): def test_timestamp(self): t = Timestamp(123, 456) self.assertEqual(t.time, 123) self.assertEqual(t.inc, 456) self.assertTrue(isinstance(t, Timestamp)) def test_datetime(self): d = datetime.datetime(2010, 5, 5, tzinfo=utc) t = Timestamp(d, 0) self.assertEqual(1273017600, t.time) self.assertEqual(d, t.as_datetime()) def test_datetime_copy_pickle(self): d = datetime.datetime(2010, 5, 5, tzinfo=utc) t = Timestamp(d, 0) dc = copy.deepcopy(d) self.assertEqual(dc, t.as_datetime()) for protocol in [0, 1, 2, -1]: pkl = pickle.dumps(d, protocol=protocol) dp = pickle.loads(pkl) self.assertEqual(dp, t.as_datetime()) def test_exceptions(self): self.assertRaises(TypeError, Timestamp) self.assertRaises(TypeError, Timestamp, None, 123) self.assertRaises(TypeError, Timestamp, 1.2, 123) self.assertRaises(TypeError, Timestamp, 123, None) self.assertRaises(TypeError, Timestamp, 123, 1.2) self.assertRaises(ValueError, Timestamp, 0, -1) self.assertRaises(ValueError, Timestamp, -1, 0) self.assertTrue(Timestamp(0, 0)) def test_equality(self): t = Timestamp(1, 1) self.assertNotEqual(t, Timestamp(0, 1)) self.assertNotEqual(t, Timestamp(1, 0)) self.assertEqual(t, Timestamp(1, 1)) # Explicitly test inequality self.assertFalse(t != Timestamp(1, 1)) def test_hash(self): self.assertEqual(hash(Timestamp(1, 2)), hash(Timestamp(1, 2))) self.assertNotEqual(hash(Timestamp(1, 2)), hash(Timestamp(1, 3))) self.assertNotEqual(hash(Timestamp(1, 2)), hash(Timestamp(2, 2))) def test_repr(self): t = Timestamp(0, 0) self.assertEqual(repr(t), "Timestamp(0, 0)") if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/__init__.py0000644000076600000240000005672513245621354016547 0ustar shanestaff00000000000000# Copyright 2010-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test suite for pymongo, bson, and gridfs. """ import os import socket import sys import time import warnings try: import ipaddress HAVE_IPADDRESS = True except ImportError: HAVE_IPADDRESS = False if sys.version_info[:2] == (2, 6): import unittest2 as unittest from unittest2 import SkipTest else: import unittest from unittest import SkipTest from functools import wraps import pymongo import pymongo.errors from bson.son import SON from bson.py3compat import _unicode from pymongo import common from pymongo.common import partition_node from pymongo.ssl_support import HAVE_SSL, validate_cert_reqs from test.version import Version if HAVE_SSL: import ssl # The host and port of a single mongod or mongos, or the seed host # for a replica set. Hostnames retrieved from isMaster will be of # unicode type in Python 2, so ensure these hostnames are unicodes, # too. It makes tests like `test_repr` predictable. host = _unicode(os.environ.get("DB_IP", 'localhost')) port = int(os.environ.get("DB_PORT", 27017)) db_user = _unicode(os.environ.get("DB_USER", "user")) db_pwd = _unicode(os.environ.get("DB_PASSWORD", "password")) CERT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'certificates') CLIENT_PEM = os.environ.get('CLIENT_PEM', os.path.join(CERT_PATH, 'client.pem')) CA_PEM = os.environ.get('CA_PEM', os.path.join(CERT_PATH, 'ca.pem')) CERT_REQS = validate_cert_reqs('CERT_REQS', os.environ.get('CERT_REQS')) _SSL_OPTIONS = dict(ssl=True) if CLIENT_PEM: _SSL_OPTIONS['ssl_certfile'] = CLIENT_PEM if CA_PEM: _SSL_OPTIONS['ssl_ca_certs'] = CA_PEM if CERT_REQS is not None: _SSL_OPTIONS['ssl_cert_reqs'] = CERT_REQS def is_server_resolvable(): """Returns True if 'server' is resolvable.""" socket_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(1) try: try: socket.gethostbyname('server') return True except socket.error: return False finally: socket.setdefaulttimeout(socket_timeout) def _connect(host, port, **kwargs): client = pymongo.MongoClient(host, port, **kwargs) start = time.time() # Jython takes a long time to connect. if sys.platform.startswith('java'): time_limit = 10 else: time_limit = .5 while not client.nodes: time.sleep(0.05) if time.time() - start > time_limit: return None return client def _create_user(authdb, user, pwd=None, roles=None, **kwargs): cmd = SON([('createUser', user)]) # X509 doesn't use a password if pwd: cmd['pwd'] = pwd cmd['roles'] = roles or ['root'] cmd.update(**kwargs) return authdb.command(cmd) class client_knobs(object): def __init__( self, heartbeat_frequency=None, min_heartbeat_interval=None, kill_cursor_frequency=None, events_queue_frequency=None): self.heartbeat_frequency = heartbeat_frequency self.min_heartbeat_interval = min_heartbeat_interval self.kill_cursor_frequency = kill_cursor_frequency self.events_queue_frequency = events_queue_frequency self.old_heartbeat_frequency = None self.old_min_heartbeat_interval = None self.old_kill_cursor_frequency = None self.old_events_queue_frequency = None def enable(self): self.old_heartbeat_frequency = common.HEARTBEAT_FREQUENCY self.old_min_heartbeat_interval = common.MIN_HEARTBEAT_INTERVAL self.old_kill_cursor_frequency = common.KILL_CURSOR_FREQUENCY self.old_events_queue_frequency = common.EVENTS_QUEUE_FREQUENCY if self.heartbeat_frequency is not None: common.HEARTBEAT_FREQUENCY = self.heartbeat_frequency if self.min_heartbeat_interval is not None: common.MIN_HEARTBEAT_INTERVAL = self.min_heartbeat_interval if self.kill_cursor_frequency is not None: common.KILL_CURSOR_FREQUENCY = self.kill_cursor_frequency if self.events_queue_frequency is not None: common.EVENTS_QUEUE_FREQUENCY = self.events_queue_frequency def __enter__(self): self.enable() def disable(self): common.HEARTBEAT_FREQUENCY = self.old_heartbeat_frequency common.MIN_HEARTBEAT_INTERVAL = self.old_min_heartbeat_interval common.KILL_CURSOR_FREQUENCY = self.old_kill_cursor_frequency common.EVENTS_QUEUE_FREQUENCY = self.old_events_queue_frequency def __exit__(self, exc_type, exc_val, exc_tb): self.disable() def _all_users(db): return set(u['user'] for u in db.command('usersInfo').get('users', [])) class ClientContext(object): def __init__(self): """Create a client and grab essential information from the server.""" self.connection_attempts = [] self.connected = False self.ismaster = {} self.w = None self.nodes = set() self.replica_set_name = None self.cmd_line = None self.version = Version(-1) # Needs to be comparable with Version self.auth_enabled = False self.test_commands_enabled = False self.is_mongos = False self.is_rs = False self.has_ipv6 = False self.ssl = False self.ssl_cert_none = False self.ssl_certfile = False self.server_is_resolvable = is_server_resolvable() self.ssl_client_options = {} self.sessions_enabled = False self.client = self._connect(host, port) if HAVE_SSL and not self.client: # Is MongoDB configured for SSL? self.client = self._connect(host, port, **_SSL_OPTIONS) if self.client: self.ssl = True self.ssl_client_options = _SSL_OPTIONS self.ssl_certfile = True if _SSL_OPTIONS.get('ssl_cert_reqs') == ssl.CERT_NONE: self.ssl_cert_none = True if self.client: self.connected = True try: self.cmd_line = self.client.admin.command('getCmdLineOpts') except pymongo.errors.OperationFailure as e: msg = e.details.get('errmsg', '') if e.code == 13 or 'unauthorized' in msg or 'login' in msg: # Unauthorized. self.auth_enabled = True else: raise else: self.auth_enabled = self._server_started_with_auth() if self.auth_enabled: # See if db_user already exists. if not self._check_user_provided(): _create_user(self.client.admin, db_user, db_pwd) self.client = self._connect( host, port, username=db_user, password=db_pwd, replicaSet=self.replica_set_name, **self.ssl_client_options) # May not have this if OperationFailure was raised earlier. self.cmd_line = self.client.admin.command('getCmdLineOpts') self.ismaster = ismaster = self.client.admin.command('isMaster') self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster if 'setName' in ismaster: self.replica_set_name = ismaster['setName'] self.is_rs = True if self.auth_enabled: # It doesn't matter which member we use as the seed here. self.client = pymongo.MongoClient( host, port, username=db_user, password=db_pwd, replicaSet=self.replica_set_name, **self.ssl_client_options) else: self.client = pymongo.MongoClient( host, port, replicaSet=self.replica_set_name, **self.ssl_client_options) # Get the authoritative ismaster result from the primary. self.ismaster = self.client.admin.command('ismaster') nodes = [partition_node(node.lower()) for node in self.ismaster.get('hosts', [])] nodes.extend([partition_node(node.lower()) for node in self.ismaster.get('passives', [])]) nodes.extend([partition_node(node.lower()) for node in self.ismaster.get('arbiters', [])]) self.nodes = set(nodes) else: self.ismaster = ismaster self.nodes = set([(host, port)]) self.w = len(self.ismaster.get("hosts", [])) or 1 self.version = Version.from_client(self.client) if 'enableTestCommands=1' in self.cmd_line['argv']: self.test_commands_enabled = True elif 'parsed' in self.cmd_line: params = self.cmd_line['parsed'].get('setParameter', []) if 'enableTestCommands=1' in params: self.test_commands_enabled = True else: params = self.cmd_line['parsed'].get('setParameter', {}) if params.get('enableTestCommands') == '1': self.test_commands_enabled = True self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid') self.has_ipv6 = self._server_started_with_ipv6() def _connect(self, host, port, **kwargs): # Jython takes a long time to connect. if sys.platform.startswith('java'): timeout_ms = 10000 else: timeout_ms = 500 client = pymongo.MongoClient( host, port, serverSelectionTimeoutMS=timeout_ms, **kwargs) try: try: client.admin.command('isMaster') # Can we connect? except pymongo.errors.OperationFailure as exc: # SERVER-32063 self.connection_attempts.append( 'connected client %r, but isMaster failed: %s' % ( client, exc)) else: self.connection_attempts.append( 'successfully connected client %r' % (client,)) # If connected, then return client with default timeout return pymongo.MongoClient(host, port, **kwargs) except pymongo.errors.ConnectionFailure as exc: self.connection_attempts.append( 'failed to connect client %r: %s' % (client, exc)) return None def connection_attempt_info(self): return '\n'.join(self.connection_attempts) @property def host(self): if self.is_rs: primary = self.client.primary return primary[0] if primary is not None else host return host @property def port(self): if self.is_rs: primary = self.client.primary return primary[1] if primary is not None else port return port @property def pair(self): return "%s:%d" % (self.host, self.port) @property def has_secondaries(self): if not self.client: return False return bool(len(self.client.secondaries)) def _check_user_provided(self): """Return True if db_user/db_password is already an admin user.""" client = pymongo.MongoClient( host, port, username=db_user, password=db_pwd, serverSelectionTimeoutMS=100, **self.ssl_client_options) try: return db_user in _all_users(client.admin) except pymongo.errors.OperationFailure as e: msg = e.details.get('errmsg', '') if e.code == 18 or 'auth fails' in msg: # Auth failed. return False else: raise def _server_started_with_auth(self): # MongoDB >= 2.0 if 'parsed' in self.cmd_line: parsed = self.cmd_line['parsed'] # MongoDB >= 2.6 if 'security' in parsed: security = parsed['security'] # >= rc3 if 'authorization' in security: return security['authorization'] == 'enabled' # < rc3 return (security.get('auth', False) or bool(security.get('keyFile'))) return parsed.get('auth', False) or bool(parsed.get('keyFile')) # Legacy argv = self.cmd_line['argv'] return '--auth' in argv or '--keyFile' in argv def _server_started_with_ipv6(self): if not socket.has_ipv6: return False if 'parsed' in self.cmd_line: if not self.cmd_line['parsed'].get('net', {}).get('ipv6'): return False else: if '--ipv6' not in self.cmd_line['argv']: return False # The server was started with --ipv6. Is there an IPv6 route to it? try: for info in socket.getaddrinfo(self.host, self.port): if info[0] == socket.AF_INET6: return True except socket.error: pass return False def _require(self, condition, msg, func=None): def make_wrapper(f): @wraps(f) def wrap(*args, **kwargs): # Always raise SkipTest if we can't connect to MongoDB if not self.connected: raise SkipTest( "Cannot connect to MongoDB on %s" % (self.pair,)) if condition: return f(*args, **kwargs) raise SkipTest(msg) return wrap if func is None: def decorate(f): return make_wrapper(f) return decorate return make_wrapper(func) def create_user(self, dbname, user, pwd=None, roles=None, **kwargs): kwargs['writeConcern'] = {'w': self.w} return _create_user(self.client[dbname], user, pwd, roles, **kwargs) def drop_user(self, dbname, user): self.client[dbname].command( 'dropUser', user, writeConcern={'w': self.w}) def require_connection(self, func): """Run a test only if we can connect to MongoDB.""" return self._require( self.connected, "Cannot connect to MongoDB on %s" % (self.pair,), func=func) def require_version_min(self, *ver): """Run a test only if the server version is at least ``version``.""" other_version = Version(*ver) return self._require(self.version >= other_version, "Server version must be at least %s" % str(other_version)) def require_version_max(self, *ver): """Run a test only if the server version is at most ``version``.""" other_version = Version(*ver) return self._require(self.version <= other_version, "Server version must be at most %s" % str(other_version)) def require_auth(self, func): """Run a test only if the server is running with auth enabled.""" return self.check_auth_with_sharding( self._require(self.auth_enabled, "Authentication is not enabled on the server", func=func)) def require_no_auth(self, func): """Run a test only if the server is running without auth enabled.""" return self._require(not self.auth_enabled, "Authentication must not be enabled on the server", func=func) def require_replica_set(self, func): """Run a test only if the client is connected to a replica set.""" return self._require(self.is_rs, "Not connected to a replica set", func=func) def require_secondaries_count(self, count): """Run a test only if the client is connected to a replica set that has `count` secondaries. """ sec_count = 0 if not self.client else len(self.client.secondaries) return self._require(sec_count >= count, "Need %d secondaries, %d available" % (count, sec_count)) def require_no_replica_set(self, func): """Run a test if the client is *not* connected to a replica set.""" return self._require( not self.is_rs, "Connected to a replica set, not a standalone mongod", func=func) def require_ipv6(self, func): """Run a test only if the client can connect to a server via IPv6.""" return self._require(self.has_ipv6, "No IPv6", func=func) def require_no_mongos(self, func): """Run a test only if the client is not connected to a mongos.""" return self._require(not self.is_mongos, "Must be connected to a mongod, not a mongos", func=func) def require_mongos(self, func): """Run a test only if the client is connected to a mongos.""" return self._require(self.is_mongos, "Must be connected to a mongos", func=func) def require_standalone(self, func): """Run a test only if the client is connected to a standalone.""" return self._require(not (self.is_mongos or self.is_rs), "Must be connected to a standalone", func=func) def require_no_standalone(self, func): """Run a test only if the client is not connected to a standalone.""" return self._require(self.is_mongos or self.is_rs, "Must be connected to a replica set or mongos", func=func) def check_auth_with_sharding(self, func): """Skip a test when connected to mongos < 2.0 and running with auth.""" condition = not (self.auth_enabled and self.is_mongos and self.version < (2,)) return self._require(condition, "Auth with sharding requires MongoDB >= 2.0.0", func=func) def require_test_commands(self, func): """Run a test only if the server has test commands enabled.""" return self._require(self.test_commands_enabled, "Test commands must be enabled", func=func) def require_ssl(self, func): """Run a test only if the client can connect over SSL.""" return self._require(self.ssl, "Must be able to connect via SSL", func=func) def require_no_ssl(self, func): """Run a test only if the client can connect over SSL.""" return self._require(not self.ssl, "Must be able to connect without SSL", func=func) def require_ssl_cert_none(self, func): """Run a test only if the client can connect with ssl.CERT_NONE.""" return self._require(self.ssl_cert_none, "Must be able to connect with ssl.CERT_NONE", func=func) def require_ssl_certfile(self, func): """Run a test only if the client can connect with ssl_certfile.""" return self._require(self.ssl_certfile, "Must be able to connect with ssl_certfile", func=func) def require_server_resolvable(self, func): """Run a test only if the hostname 'server' is resolvable.""" return self._require(self.server_is_resolvable, "No hosts entry for 'server'. Cannot validate " "hostname in the certificate", func=func) def require_sessions(self, func): """Run a test only if the deployment supports sessions.""" return self._require(self.sessions_enabled, "Sessions not supported", func=func) # Reusable client context client_context = ClientContext() def sanitize_cmd(cmd): cp = cmd.copy() cp.pop('$clusterTime', None) cp.pop('lsid', None) return cp def sanitize_reply(reply): cp = reply.copy() cp.pop('$clusterTime', None) cp.pop('operationTime', None) return cp class PyMongoTestCase(unittest.TestCase): def assertEqualCommand(self, expected, actual, msg=None): self.assertEqual(expected, sanitize_cmd(actual), msg) def assertEqualReply(self, expected, actual, msg=None): self.assertEqual(expected, sanitize_reply(actual), msg) class IntegrationTest(PyMongoTestCase): """Base class for TestCases that need a connection to MongoDB to pass.""" @classmethod @client_context.require_connection def setUpClass(cls): cls.client = client_context.client cls.db = cls.client.pymongo_test if client_context.auth_enabled: cls.credentials = {'username': db_user, 'password': db_pwd} else: cls.credentials = {} # Use assertRaisesRegex if available, otherwise use Python 2.7's # deprecated assertRaisesRegexp, with a 'p'. if not hasattr(unittest.TestCase, 'assertRaisesRegex'): IntegrationTest.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp class MockClientTest(unittest.TestCase): """Base class for TestCases that use MockClient. This class is *not* an IntegrationTest: if properly written, MockClient tests do not require a running server. The class temporarily overrides HEARTBEAT_FREQUENCY to speed up tests. """ def setUp(self): super(MockClientTest, self).setUp() self.client_knobs = client_knobs( heartbeat_frequency=0.001, min_heartbeat_interval=0.001) self.client_knobs.enable() def tearDown(self): self.client_knobs.disable() super(MockClientTest, self).tearDown() def setup(): warnings.resetwarnings() warnings.simplefilter("always") def teardown(): c = client_context.client c.drop_database("pymongo-pooling-tests") c.drop_database("pymongo_test") c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") c.drop_database("pymongo_test_mike") c.drop_database("pymongo_test_bernie") class PymongoTestRunner(unittest.TextTestRunner): def run(self, test): setup() result = super(PymongoTestRunner, self).run(test) try: teardown() finally: return result def test_cases(suite): """Iterator over all TestCases within a TestSuite.""" for suite_or_case in suite._tests: if isinstance(suite_or_case, unittest.TestCase): # unittest.TestCase yield suite_or_case else: # unittest.TestSuite for case in test_cases(suite_or_case): yield case pymongo-3.6.1/test/test_raw_bson.py0000644000076600000240000001334713245621354017652 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import uuid from bson import BSON from bson.binary import JAVA_LEGACY from bson.codec_options import CodecOptions from bson.raw_bson import RawBSONDocument from test import client_context, unittest class TestRawBSONDocument(unittest.TestCase): # {u'_id': ObjectId('556df68b6e32ab21a95e0785'), # u'name': u'Sherlock', # u'addresses': [{u'street': u'Baker Street'}]} bson_string = ( b'Z\x00\x00\x00\x07_id\x00Um\xf6\x8bn2\xab!\xa9^\x07\x85\x02name\x00\t' b'\x00\x00\x00Sherlock\x00\x04addresses\x00&\x00\x00\x00\x030\x00\x1e' b'\x00\x00\x00\x02street\x00\r\x00\x00\x00Baker Street\x00\x00\x00\x00' ) document = RawBSONDocument(bson_string) @classmethod def setUpClass(cls): cls.client = client_context.client def tearDown(self): if client_context.connected: self.client.pymongo_test.test_raw.drop() def test_decode(self): self.assertEqual('Sherlock', self.document['name']) first_address = self.document['addresses'][0] self.assertIsInstance(first_address, RawBSONDocument) self.assertEqual('Baker Street', first_address['street']) def test_raw(self): self.assertEqual(self.bson_string, self.document.raw) @client_context.require_connection def test_round_trip(self): db = self.client.get_database( 'pymongo_test', codec_options=CodecOptions(document_class=RawBSONDocument)) db.test_raw.insert_one(self.document) result = db.test_raw.find_one(self.document['_id']) self.assertIsInstance(result, RawBSONDocument) self.assertEqual(dict(self.document.items()), dict(result.items())) def test_with_codec_options(self): # {u'date': datetime.datetime(2015, 6, 3, 18, 40, 50, 826000), # u'_id': UUID('026fab8f-975f-4965-9fbf-85ad874c60ff')} # encoded with JAVA_LEGACY uuid representation. bson_string = ( b'-\x00\x00\x00\x05_id\x00\x10\x00\x00\x00\x03eI_\x97\x8f\xabo\x02' b'\xff`L\x87\xad\x85\xbf\x9f\tdate\x00\x8a\xd6\xb9\xbaM' b'\x01\x00\x00\x00' ) document = RawBSONDocument( bson_string, codec_options=CodecOptions(uuid_representation=JAVA_LEGACY, document_class=RawBSONDocument)) self.assertEqual(uuid.UUID('026fab8f-975f-4965-9fbf-85ad874c60ff'), document['_id']) @client_context.require_connection def test_round_trip_codec_options(self): doc = { 'date': datetime.datetime(2015, 6, 3, 18, 40, 50, 826000), '_id': uuid.UUID('026fab8f-975f-4965-9fbf-85ad874c60ff') } db = self.client.pymongo_test coll = db.get_collection( 'test_raw', codec_options=CodecOptions(uuid_representation=JAVA_LEGACY)) coll.insert_one(doc) raw_java_legacy = CodecOptions(uuid_representation=JAVA_LEGACY, document_class=RawBSONDocument) coll = db.get_collection('test_raw', codec_options=raw_java_legacy) self.assertEqual( RawBSONDocument(BSON.encode(doc, codec_options=raw_java_legacy)), coll.find_one()) @client_context.require_connection def test_raw_bson_document_embedded(self): doc = {'embedded': self.document} db = self.client.pymongo_test db.test_raw.insert_one(doc) result = db.test_raw.find_one() self.assertEqual(BSON(self.document.raw).decode(), result['embedded']) # Make sure that CodecOptions are preserved. # {'embedded': [ # {u'date': datetime.datetime(2015, 6, 3, 18, 40, 50, 826000), # u'_id': UUID('026fab8f-975f-4965-9fbf-85ad874c60ff')} # ]} # encoded with JAVA_LEGACY uuid representation. bson_string = ( b'D\x00\x00\x00\x04embedded\x005\x00\x00\x00\x030\x00-\x00\x00\x00' b'\tdate\x00\x8a\xd6\xb9\xbaM\x01\x00\x00\x05_id\x00\x10\x00\x00' b'\x00\x03eI_\x97\x8f\xabo\x02\xff`L\x87\xad\x85\xbf\x9f\x00\x00' b'\x00' ) rbd = RawBSONDocument( bson_string, codec_options=CodecOptions(uuid_representation=JAVA_LEGACY, document_class=RawBSONDocument)) db.test_raw.drop() db.test_raw.insert_one(rbd) result = db.get_collection('test_raw', codec_options=CodecOptions( uuid_representation=JAVA_LEGACY)).find_one() self.assertEqual(rbd['embedded'][0]['_id'], result['embedded'][0]['_id']) @client_context.require_connection def test_write_response_raw_bson(self): coll = self.client.get_database( 'pymongo_test', codec_options=CodecOptions(document_class=RawBSONDocument)).test_raw # No Exceptions raised while handling write response. coll.insert_one(self.document) coll.delete_one(self.document) coll.insert_many([self.document]) coll.delete_many(self.document) coll.update_one(self.document, {'$set': {'a': 'b'}}, upsert=True) coll.update_many(self.document, {'$set': {'b': 'c'}}) pymongo-3.6.1/test/test_collection.py0000644000076600000240000026525313245621354020200 0ustar shanestaff00000000000000# -*- coding: utf-8 -*- # Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the collection module.""" import contextlib import re import sys import threading from codecs import utf_8_decode from collections import defaultdict sys.path[0:0] = [""] import bson from bson.raw_bson import RawBSONDocument from bson.regex import Regex from bson.code import Code from bson.codec_options import CodecOptions from bson.objectid import ObjectId from bson.py3compat import itervalues from bson.son import SON from pymongo import (ASCENDING, DESCENDING, GEO2D, GEOHAYSTACK, GEOSPHERE, HASHED, TEXT) from pymongo import monitoring from pymongo.bulk import BulkWriteError from pymongo.collection import Collection, ReturnDocument from pymongo.command_cursor import CommandCursor from pymongo.cursor import CursorType from pymongo.errors import (ConfigurationError, DocumentTooLarge, DuplicateKeyError, ExecutionTimeout, InvalidDocument, InvalidName, InvalidOperation, OperationFailure, WriteConcernError) from pymongo.message import _COMMAND_OVERHEAD, _gen_find_command from pymongo.mongo_client import MongoClient from pymongo.operations import * from pymongo.read_concern import DEFAULT_READ_CONCERN from pymongo.read_preferences import ReadPreference from pymongo.results import (InsertOneResult, InsertManyResult, UpdateResult, DeleteResult) from pymongo.write_concern import WriteConcern from test.test_client import IntegrationTest from test.utils import (is_mongos, get_pool, rs_or_single_client, single_client, wait_until, EventListener, IMPOSSIBLE_WRITE_CONCERN) from test import client_context, unittest class TestCollectionNoConnect(unittest.TestCase): """Test Collection features on a client that does not connect. """ @classmethod def setUpClass(cls): cls.db = MongoClient(connect=False).pymongo_test def test_collection(self): self.assertRaises(TypeError, Collection, self.db, 5) def make_col(base, name): return base[name] self.assertRaises(InvalidName, make_col, self.db, "") self.assertRaises(InvalidName, make_col, self.db, "te$t") self.assertRaises(InvalidName, make_col, self.db, ".test") self.assertRaises(InvalidName, make_col, self.db, "test.") self.assertRaises(InvalidName, make_col, self.db, "tes..t") self.assertRaises(InvalidName, make_col, self.db.test, "") self.assertRaises(InvalidName, make_col, self.db.test, "te$t") self.assertRaises(InvalidName, make_col, self.db.test, ".test") self.assertRaises(InvalidName, make_col, self.db.test, "test.") self.assertRaises(InvalidName, make_col, self.db.test, "tes..t") self.assertRaises(InvalidName, make_col, self.db.test, "tes\x00t") def test_getattr(self): coll = self.db.test self.assertTrue(isinstance(coll['_does_not_exist'], Collection)) with self.assertRaises(AttributeError) as context: coll._does_not_exist # Message should be: # "AttributeError: Collection has no attribute '_does_not_exist'. To # access the test._does_not_exist collection, use # database['test._does_not_exist']." self.assertIn("has no attribute '_does_not_exist'", str(context.exception)) def test_iteration(self): self.assertRaises(TypeError, next, self.db) class TestCollection(IntegrationTest): @classmethod def setUpClass(cls): super(TestCollection, cls).setUpClass() cls.w = client_context.w @classmethod def tearDownClass(cls): cls.db.drop_collection("test_large_limit") @contextlib.contextmanager def write_concern_collection(self): if client_context.version.at_least(3, 3, 9) and client_context.is_rs: with self.assertRaises(WriteConcernError): # Unsatisfiable write concern. yield Collection( self.db, 'test', write_concern=WriteConcern(w=len(client_context.nodes) + 1)) else: yield self.db.test def test_equality(self): self.assertTrue(isinstance(self.db.test, Collection)) self.assertEqual(self.db.test, self.db["test"]) self.assertEqual(self.db.test, Collection(self.db, "test")) self.assertEqual(self.db.test.mike, self.db["test.mike"]) self.assertEqual(self.db.test["mike"], self.db["test.mike"]) @client_context.require_version_min(3, 3, 9) def test_create(self): # No Exception. db = client_context.client.pymongo_test db.create_test_no_wc.drop() wait_until(lambda: 'create_test_no_wc' not in db.collection_names(), 'drop create_test_no_wc collection') Collection(db, name='create_test_no_wc', create=True) wait_until(lambda: 'create_test_no_wc' in db.collection_names(), 'create create_test_no_wc collection') # SERVER-33317 if (not client_context.is_mongos or not client_context.version.at_least(3, 7, 0)): with self.assertRaises(OperationFailure): Collection( db, name='create-test-wc', write_concern=IMPOSSIBLE_WRITE_CONCERN, create=True) def test_drop_nonexistent_collection(self): self.db.drop_collection('test') self.assertFalse('test' in self.db.collection_names()) # No exception self.db.drop_collection('test') def test_create_indexes(self): db = self.db self.assertRaises(TypeError, db.test.create_indexes, 'foo') self.assertRaises(TypeError, db.test.create_indexes, ['foo']) self.assertRaises(TypeError, IndexModel, 5) self.assertRaises(ValueError, IndexModel, []) db.test.drop_indexes() db.test.insert_one({}) self.assertEqual(len(db.test.index_information()), 1) db.test.create_indexes([IndexModel("hello")]) db.test.create_indexes([IndexModel([("hello", DESCENDING), ("world", ASCENDING)])]) # Tuple instead of list. db.test.create_indexes([IndexModel((("world", ASCENDING),))]) self.assertEqual(len(db.test.index_information()), 4) db.test.drop_indexes() names = db.test.create_indexes([IndexModel([("hello", DESCENDING), ("world", ASCENDING)], name="hello_world")]) self.assertEqual(names, ["hello_world"]) db.test.drop_indexes() self.assertEqual(len(db.test.index_information()), 1) db.test.create_indexes([IndexModel("hello")]) self.assertTrue("hello_1" in db.test.index_information()) db.test.drop_indexes() self.assertEqual(len(db.test.index_information()), 1) names = db.test.create_indexes([IndexModel([("hello", DESCENDING), ("world", ASCENDING)]), IndexModel("hello")]) info = db.test.index_information() for name in names: self.assertTrue(name in info) db.test.drop() db.test.insert_one({'a': 1}) db.test.insert_one({'a': 1}) self.assertRaises( DuplicateKeyError, db.test.create_indexes, [IndexModel('a', unique=True)]) with self.write_concern_collection() as coll: coll.create_indexes([IndexModel('hello')]) def test_create_index(self): db = self.db self.assertRaises(TypeError, db.test.create_index, 5) self.assertRaises(TypeError, db.test.create_index, {"hello": 1}) self.assertRaises(ValueError, db.test.create_index, []) db.test.drop_indexes() db.test.insert_one({}) self.assertEqual(len(db.test.index_information()), 1) db.test.create_index("hello") db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)]) # Tuple instead of list. db.test.create_index((("world", ASCENDING),)) self.assertEqual(len(db.test.index_information()), 4) db.test.drop_indexes() ix = db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)], name="hello_world") self.assertEqual(ix, "hello_world") db.test.drop_indexes() self.assertEqual(len(db.test.index_information()), 1) db.test.create_index("hello") self.assertTrue("hello_1" in db.test.index_information()) db.test.drop_indexes() self.assertEqual(len(db.test.index_information()), 1) db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)]) self.assertTrue("hello_-1_world_1" in db.test.index_information()) db.test.drop() db.test.insert_one({'a': 1}) db.test.insert_one({'a': 1}) self.assertRaises( DuplicateKeyError, db.test.create_index, 'a', unique=True) with self.write_concern_collection() as coll: coll.create_index([('hello', DESCENDING)]) def test_drop_index(self): db = self.db db.test.drop_indexes() db.test.create_index("hello") name = db.test.create_index("goodbye") self.assertEqual(len(db.test.index_information()), 3) self.assertEqual(name, "goodbye_1") db.test.drop_index(name) # Drop it again. with self.assertRaises(OperationFailure): db.test.drop_index(name) self.assertEqual(len(db.test.index_information()), 2) self.assertTrue("hello_1" in db.test.index_information()) db.test.drop_indexes() db.test.create_index("hello") name = db.test.create_index("goodbye") self.assertEqual(len(db.test.index_information()), 3) self.assertEqual(name, "goodbye_1") db.test.drop_index([("goodbye", ASCENDING)]) self.assertEqual(len(db.test.index_information()), 2) self.assertTrue("hello_1" in db.test.index_information()) with self.write_concern_collection() as coll: coll.drop_index('hello_1') @client_context.require_no_mongos @client_context.require_test_commands def test_index_management_max_time_ms(self): if (client_context.version[:2] == (3, 4) and client_context.version[2] < 4): raise unittest.SkipTest("SERVER-27711") coll = self.db.test self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") try: self.assertRaises( ExecutionTimeout, coll.create_index, "foo", maxTimeMS=1) self.assertRaises( ExecutionTimeout, coll.create_indexes, [IndexModel("foo")], maxTimeMS=1) self.assertRaises( ExecutionTimeout, coll.drop_index, "foo", maxTimeMS=1) self.assertRaises( ExecutionTimeout, coll.drop_indexes, maxTimeMS=1) self.assertRaises( ExecutionTimeout, coll.reindex, maxTimeMS=1) finally: self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") def test_reindex(self): db = self.db db.drop_collection("test") db.test.insert_one({"foo": "bar", "who": "what", "when": "how"}) db.test.create_index("foo") db.test.create_index("who") db.test.create_index("when") info = db.test.index_information() def check_result(result): self.assertEqual(4, result['nIndexes']) indexes = result['indexes'] names = [idx['name'] for idx in indexes] for name in names: self.assertTrue(name in info) for key in info: self.assertTrue(key in names) reindexed = db.test.reindex() if 'raw' in reindexed: # mongos for result in itervalues(reindexed['raw']): check_result(result) else: check_result(reindexed) coll = Collection( self.db, 'test', write_concern=WriteConcern(w=100)) # No error since writeConcern is not sent. coll.reindex() def test_list_indexes(self): db = self.db db.test.drop() db.test.insert_one({}) # create collection def map_indexes(indexes): return dict([(index["name"], index) for index in indexes]) indexes = list(db.test.list_indexes()) self.assertEqual(len(indexes), 1) self.assertTrue("_id_" in map_indexes(indexes)) db.test.create_index("hello") indexes = list(db.test.list_indexes()) self.assertEqual(len(indexes), 2) self.assertEqual(map_indexes(indexes)["hello_1"]["key"], SON([("hello", ASCENDING)])) db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)], unique=True) indexes = list(db.test.list_indexes()) self.assertEqual(len(indexes), 3) index_map = map_indexes(indexes) self.assertEqual(index_map["hello_-1_world_1"]["key"], SON([("hello", DESCENDING), ("world", ASCENDING)])) self.assertEqual(True, index_map["hello_-1_world_1"]["unique"]) # List indexes on a collection that does not exist. indexes = list(db.does_not_exist.list_indexes()) self.assertEqual(len(indexes), 0) # List indexes on a database that does not exist. indexes = list(self.client.db_does_not_exist.coll.list_indexes()) self.assertEqual(len(indexes), 0) def test_index_info(self): db = self.db db.test.drop() db.test.insert_one({}) # create collection self.assertEqual(len(db.test.index_information()), 1) self.assertTrue("_id_" in db.test.index_information()) db.test.create_index("hello") self.assertEqual(len(db.test.index_information()), 2) self.assertEqual(db.test.index_information()["hello_1"]["key"], [("hello", ASCENDING)]) db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)], unique=True) self.assertEqual(db.test.index_information()["hello_1"]["key"], [("hello", ASCENDING)]) self.assertEqual(len(db.test.index_information()), 3) self.assertEqual([("hello", DESCENDING), ("world", ASCENDING)], db.test.index_information()["hello_-1_world_1"]["key"] ) self.assertEqual( True, db.test.index_information()["hello_-1_world_1"]["unique"]) def test_index_geo2d(self): db = self.db db.test.drop_indexes() self.assertEqual('loc_2d', db.test.create_index([("loc", GEO2D)])) index_info = db.test.index_information()['loc_2d'] self.assertEqual([('loc', '2d')], index_info['key']) @client_context.require_no_mongos def test_index_haystack(self): db = self.db db.test.drop() _id = db.test.insert_one({ "pos": {"long": 34.2, "lat": 33.3}, "type": "restaurant" }).inserted_id db.test.insert_one({ "pos": {"long": 34.2, "lat": 37.3}, "type": "restaurant" }) db.test.insert_one({ "pos": {"long": 59.1, "lat": 87.2}, "type": "office" }) db.test.create_index( [("pos", GEOHAYSTACK), ("type", ASCENDING)], bucketSize=1 ) results = db.command(SON([ ("geoSearch", "test"), ("near", [33, 33]), ("maxDistance", 6), ("search", {"type": "restaurant"}), ("limit", 30), ]))['results'] self.assertEqual(2, len(results)) self.assertEqual({ "_id": _id, "pos": {"long": 34.2, "lat": 33.3}, "type": "restaurant" }, results[0]) @client_context.require_no_mongos def test_index_text(self): db = self.db db.test.drop_indexes() self.assertEqual("t_text", db.test.create_index([("t", TEXT)])) index_info = db.test.index_information()["t_text"] self.assertTrue("weights" in index_info) db.test.insert_many([ {'t': 'spam eggs and spam'}, {'t': 'spam'}, {'t': 'egg sausage and bacon'}]) # MongoDB 2.6 text search. Create 'score' field in projection. cursor = db.test.find( {'$text': {'$search': 'spam'}}, {'score': {'$meta': 'textScore'}}) # Sort by 'score' field. cursor.sort([('score', {'$meta': 'textScore'})]) results = list(cursor) self.assertTrue(results[0]['score'] >= results[1]['score']) db.test.drop_indexes() def test_index_2dsphere(self): db = self.db db.test.drop_indexes() self.assertEqual("geo_2dsphere", db.test.create_index([("geo", GEOSPHERE)])) for dummy, info in db.test.index_information().items(): field, idx_type = info['key'][0] if field == 'geo' and idx_type == '2dsphere': break else: self.fail("2dsphere index not found.") poly = {"type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]} query = {"geo": {"$within": {"$geometry": poly}}} # This query will error without a 2dsphere index. db.test.find(query) db.test.drop_indexes() def test_index_hashed(self): db = self.db db.test.drop_indexes() self.assertEqual("a_hashed", db.test.create_index([("a", HASHED)])) for dummy, info in db.test.index_information().items(): field, idx_type = info['key'][0] if field == 'a' and idx_type == 'hashed': break else: self.fail("hashed index not found.") db.test.drop_indexes() def test_index_sparse(self): db = self.db db.test.drop_indexes() db.test.create_index([('key', ASCENDING)], sparse=True) self.assertTrue(db.test.index_information()['key_1']['sparse']) def test_index_background(self): db = self.db db.test.drop_indexes() db.test.create_index([('keya', ASCENDING)]) db.test.create_index([('keyb', ASCENDING)], background=False) db.test.create_index([('keyc', ASCENDING)], background=True) self.assertFalse('background' in db.test.index_information()['keya_1']) self.assertFalse(db.test.index_information()['keyb_1']['background']) self.assertTrue(db.test.index_information()['keyc_1']['background']) def _drop_dups_setup(self, db): db.drop_collection('test') db.test.insert_one({'i': 1}) db.test.insert_one({'i': 2}) db.test.insert_one({'i': 2}) # duplicate db.test.insert_one({'i': 3}) @client_context.require_version_max(2, 6) def test_index_drop_dups(self): # Try dropping duplicates db = self.db self._drop_dups_setup(db) # No error, just drop the duplicate db.test.create_index([('i', ASCENDING)], unique=True, dropDups=True) # Duplicate was dropped self.assertEqual(3, db.test.count()) # Index was created, plus the index on _id self.assertEqual(2, len(db.test.index_information())) def test_index_dont_drop_dups(self): # Try *not* dropping duplicates db = self.db self._drop_dups_setup(db) # There's a duplicate def test_create(): db.test.create_index( [('i', ASCENDING)], unique=True, dropDups=False ) self.assertRaises(DuplicateKeyError, test_create) # Duplicate wasn't dropped self.assertEqual(4, db.test.count()) # Index wasn't created, only the default index on _id self.assertEqual(1, len(db.test.index_information())) # Get the plan dynamically because the explain format will change. def get_plan_stage(self, root, stage): if root.get('stage') == stage: return root elif "inputStage" in root: return self.get_plan_stage(root['inputStage'], stage) elif "inputStages" in root: for i in root['inputStages']: stage = self.get_plan_stage(i, stage) if stage: return stage elif "shards" in root: for i in root['shards']: stage = self.get_plan_stage(i['winningPlan'], stage) if stage: return stage return {} @client_context.require_version_min(3, 1, 9, -1) def test_index_filter(self): db = self.db db.drop_collection("test") # Test bad filter spec on create. self.assertRaises(OperationFailure, db.test.create_index, "x", partialFilterExpression=5) self.assertRaises(OperationFailure, db.test.create_index, "x", partialFilterExpression={"x": {"$asdasd": 3}}) self.assertRaises(OperationFailure, db.test.create_index, "x", partialFilterExpression={"$and": 5}) self.assertRaises(OperationFailure, db.test.create_index, "x", partialFilterExpression={ "$and": [{"$and": [{"x": {"$lt": 2}}, {"x": {"$gt": 0}}]}, {"x": {"$exists": True}}]}) self.assertEqual("x_1", db.test.create_index( [('x', ASCENDING)], partialFilterExpression={"a": {"$lte": 1.5}})) db.test.insert_one({"x": 5, "a": 2}) db.test.insert_one({"x": 6, "a": 1}) # Operations that use the partial index. explain = db.test.find({"x": 6, "a": 1}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'IXSCAN') self.assertEqual("x_1", stage.get('indexName')) self.assertTrue(stage.get('isPartial')) explain = db.test.find({"x": {"$gt": 1}, "a": 1}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'IXSCAN') self.assertEqual("x_1", stage.get('indexName')) self.assertTrue(stage.get('isPartial')) explain = db.test.find({"x": 6, "a": {"$lte": 1}}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'IXSCAN') self.assertEqual("x_1", stage.get('indexName')) self.assertTrue(stage.get('isPartial')) # Operations that do not use the partial index. explain = db.test.find({"x": 6, "a": {"$lte": 1.6}}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'COLLSCAN') self.assertNotEqual({}, stage) explain = db.test.find({"x": 6}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'COLLSCAN') self.assertNotEqual({}, stage) # Test drop_indexes. db.test.drop_index("x_1") explain = db.test.find({"x": 6, "a": 1}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'COLLSCAN') self.assertNotEqual({}, stage) def test_field_selection(self): db = self.db db.drop_collection("test") doc = {"a": 1, "b": 5, "c": {"d": 5, "e": 10}} db.test.insert_one(doc) # Test field inclusion doc = next(db.test.find({}, ["_id"])) self.assertEqual(list(doc), ["_id"]) doc = next(db.test.find({}, ["a"])) l = list(doc) l.sort() self.assertEqual(l, ["_id", "a"]) doc = next(db.test.find({}, ["b"])) l = list(doc) l.sort() self.assertEqual(l, ["_id", "b"]) doc = next(db.test.find({}, ["c"])) l = list(doc) l.sort() self.assertEqual(l, ["_id", "c"]) doc = next(db.test.find({}, ["a"])) self.assertEqual(doc["a"], 1) doc = next(db.test.find({}, ["b"])) self.assertEqual(doc["b"], 5) doc = next(db.test.find({}, ["c"])) self.assertEqual(doc["c"], {"d": 5, "e": 10}) # Test inclusion of fields with dots doc = next(db.test.find({}, ["c.d"])) self.assertEqual(doc["c"], {"d": 5}) doc = next(db.test.find({}, ["c.e"])) self.assertEqual(doc["c"], {"e": 10}) doc = next(db.test.find({}, ["b", "c.e"])) self.assertEqual(doc["c"], {"e": 10}) doc = next(db.test.find({}, ["b", "c.e"])) l = list(doc) l.sort() self.assertEqual(l, ["_id", "b", "c"]) doc = next(db.test.find({}, ["b", "c.e"])) self.assertEqual(doc["b"], 5) # Test field exclusion doc = next(db.test.find({}, {"a": False, "b": 0})) l = list(doc) l.sort() self.assertEqual(l, ["_id", "c"]) doc = next(db.test.find({}, {"_id": False})) l = list(doc) self.assertFalse("_id" in l) def test_options(self): db = self.db db.drop_collection("test") db.create_collection("test", capped=True, size=4096) result = db.test.options() # mongos 2.2.x adds an $auth field when auth is enabled. result.pop('$auth', None) self.assertEqual(result, {"capped": True, 'size': 4096}) db.drop_collection("test") def test_insert_one(self): db = self.db db.test.drop() document = {"_id": 1000} result = db.test.insert_one(document) self.assertTrue(isinstance(result, InsertOneResult)) self.assertTrue(isinstance(result.inserted_id, int)) self.assertEqual(document["_id"], result.inserted_id) self.assertTrue(result.acknowledged) self.assertIsNotNone(db.test.find_one({"_id": document["_id"]})) self.assertEqual(1, db.test.count()) document = {"foo": "bar"} result = db.test.insert_one(document) self.assertTrue(isinstance(result, InsertOneResult)) self.assertTrue(isinstance(result.inserted_id, ObjectId)) self.assertEqual(document["_id"], result.inserted_id) self.assertTrue(result.acknowledged) self.assertIsNotNone(db.test.find_one({"_id": document["_id"]})) self.assertEqual(2, db.test.count()) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) result = db.test.insert_one(document) self.assertTrue(isinstance(result, InsertOneResult)) self.assertTrue(isinstance(result.inserted_id, ObjectId)) self.assertEqual(document["_id"], result.inserted_id) self.assertFalse(result.acknowledged) # The insert failed duplicate key... wait_until(lambda: 2 == db.test.count(), 'forcing duplicate key error') document = RawBSONDocument( bson.BSON.encode({'_id': ObjectId(), 'foo': 'bar'})) result = db.test.insert_one(document) self.assertTrue(isinstance(result, InsertOneResult)) self.assertEqual(result.inserted_id, None) def test_insert_many(self): db = self.db db.test.drop() docs = [{} for _ in range(5)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(isinstance(result.inserted_ids, list)) self.assertEqual(5, len(result.inserted_ids)) for doc in docs: _id = doc["_id"] self.assertTrue(isinstance(_id, ObjectId)) self.assertTrue(_id in result.inserted_ids) self.assertEqual(1, db.test.count({'_id': _id})) self.assertTrue(result.acknowledged) docs = [{"_id": i} for i in range(5)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(isinstance(result.inserted_ids, list)) self.assertEqual(5, len(result.inserted_ids)) for doc in docs: _id = doc["_id"] self.assertTrue(isinstance(_id, int)) self.assertTrue(_id in result.inserted_ids) self.assertEqual(1, db.test.count({"_id": _id})) self.assertTrue(result.acknowledged) docs = [RawBSONDocument(bson.BSON.encode({"_id": i + 5})) for i in range(5)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(isinstance(result.inserted_ids, list)) self.assertEqual([], result.inserted_ids) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) docs = [{} for _ in range(5)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertFalse(result.acknowledged) self.assertEqual(20, db.test.count()) def test_delete_one(self): self.db.test.drop() self.db.test.insert_one({"x": 1}) self.db.test.insert_one({"y": 1}) self.db.test.insert_one({"z": 1}) result = self.db.test.delete_one({"x": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertEqual(1, result.deleted_count) self.assertTrue(result.acknowledged) self.assertEqual(2, self.db.test.count()) result = self.db.test.delete_one({"y": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertEqual(1, result.deleted_count) self.assertTrue(result.acknowledged) self.assertEqual(1, self.db.test.count()) db = self.db.client.get_database(self.db.name, write_concern=WriteConcern(w=0)) result = db.test.delete_one({"z": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertRaises(InvalidOperation, lambda: result.deleted_count) self.assertFalse(result.acknowledged) wait_until(lambda: 0 == db.test.count(), 'delete 1 documents') def test_delete_many(self): self.db.test.drop() self.db.test.insert_one({"x": 1}) self.db.test.insert_one({"x": 1}) self.db.test.insert_one({"y": 1}) self.db.test.insert_one({"y": 1}) result = self.db.test.delete_many({"x": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertEqual(2, result.deleted_count) self.assertTrue(result.acknowledged) self.assertEqual(0, self.db.test.count({"x": 1})) db = self.db.client.get_database(self.db.name, write_concern=WriteConcern(w=0)) result = db.test.delete_many({"y": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertRaises(InvalidOperation, lambda: result.deleted_count) self.assertFalse(result.acknowledged) wait_until(lambda: 0 == db.test.count(), 'delete 2 documents') def test_command_document_too_large(self): large = '*' * (self.client.max_bson_size + _COMMAND_OVERHEAD) coll = self.db.test self.assertRaises( DocumentTooLarge, coll.insert_one, {'data': large}) # update_one and update_many are the same self.assertRaises( DocumentTooLarge, coll.replace_one, {}, {'data': large}) self.assertRaises( DocumentTooLarge, coll.delete_one, {'data': large}) @client_context.require_version_min(3, 1, 9, -1) def test_insert_bypass_document_validation(self): db = self.db db.test.drop() db.create_collection("test", validator={"a": {"$exists": True}}) db_w0 = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=0)) # Test insert_one self.assertRaises(OperationFailure, db.test.insert_one, {"_id": 1, "x": 100}) result = db.test.insert_one({"_id": 1, "x": 100}, bypass_document_validation=True) self.assertTrue(isinstance(result, InsertOneResult)) self.assertEqual(1, result.inserted_id) result = db.test.insert_one({"_id":2, "a":0}) self.assertTrue(isinstance(result, InsertOneResult)) self.assertEqual(2, result.inserted_id) self.assertRaises(OperationFailure, db_w0.test.insert_one, {"x": 1}, bypass_document_validation=True) # Test insert_many docs = [{"_id": i, "x": 100 - i} for i in range(3, 100)] self.assertRaises(OperationFailure, db.test.insert_many, docs) result = db.test.insert_many(docs, bypass_document_validation=True) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(97, len(result.inserted_ids)) for doc in docs: _id = doc["_id"] self.assertTrue(isinstance(_id, int)) self.assertTrue(_id in result.inserted_ids) self.assertEqual(1, db.test.count({"x": doc["x"]})) self.assertTrue(result.acknowledged) docs = [{"_id": i, "a": 200 - i} for i in range(100, 200)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(97, len(result.inserted_ids)) for doc in docs: _id = doc["_id"] self.assertTrue(isinstance(_id, int)) self.assertTrue(_id in result.inserted_ids) self.assertEqual(1, db.test.count({"a": doc["a"]})) self.assertTrue(result.acknowledged) self.assertRaises(OperationFailure, db_w0.test.insert_many, [{"x": 1}, {"x": 2}], bypass_document_validation=True) @client_context.require_version_min(3, 1, 9, -1) def test_replace_bypass_document_validation(self): db = self.db db.test.drop() db.create_collection("test", validator={"a": {"$exists": True}}) db_w0 = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=0)) # Test replace_one db.test.insert_one({"a": 101}) self.assertRaises(OperationFailure, db.test.replace_one, {"a": 101}, {"y": 1}) self.assertEqual(0, db.test.count({"y": 1})) self.assertEqual(1, db.test.count({"a": 101})) db.test.replace_one({"a": 101}, {"y": 1}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"a": 101})) self.assertEqual(1, db.test.count({"y": 1})) db.test.replace_one({"y": 1}, {"a": 102}) self.assertEqual(0, db.test.count({"y": 1})) self.assertEqual(0, db.test.count({"a": 101})) self.assertEqual(1, db.test.count({"a": 102})) db.test.insert_one({"y": 1}, bypass_document_validation=True) self.assertRaises(OperationFailure, db.test.replace_one, {"y": 1}, {"x": 101}) self.assertEqual(0, db.test.count({"x": 101})) self.assertEqual(1, db.test.count({"y": 1})) db.test.replace_one({"y": 1}, {"x": 101}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"y": 1})) self.assertEqual(1, db.test.count({"x": 101})) db.test.replace_one({"x": 101}, {"a": 103}, bypass_document_validation=False) self.assertEqual(0, db.test.count({"x": 101})) self.assertEqual(1, db.test.count({"a": 103})) self.assertRaises(OperationFailure, db_w0.test.replace_one, {"y": 1}, {"x": 1}, bypass_document_validation=True) @client_context.require_version_min(3, 1, 9, -1) def test_update_bypass_document_validation(self): db = self.db db.test.drop() db.test.insert_one({"z": 5}) db.command(SON([("collMod", "test"), ("validator", {"z": {"$gte": 0}})])) db_w0 = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=0)) # Test update_one self.assertRaises(OperationFailure, db.test.update_one, {"z": 5}, {"$inc": {"z": -10}}) self.assertEqual(0, db.test.count({"z": -5})) self.assertEqual(1, db.test.count({"z": 5})) db.test.update_one({"z": 5}, {"$inc": {"z": -10}}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"z": 5})) self.assertEqual(1, db.test.count({"z": -5})) db.test.update_one({"z": -5}, {"$inc": {"z": 6}}, bypass_document_validation=False) self.assertEqual(1, db.test.count({"z": 1})) self.assertEqual(0, db.test.count({"z": -5})) db.test.insert_one({"z": -10}, bypass_document_validation=True) self.assertRaises(OperationFailure, db.test.update_one, {"z": -10}, {"$inc": {"z": 1}}) self.assertEqual(0, db.test.count({"z": -9})) self.assertEqual(1, db.test.count({"z": -10})) db.test.update_one({"z": -10}, {"$inc": {"z": 1}}, bypass_document_validation=True) self.assertEqual(1, db.test.count({"z": -9})) self.assertEqual(0, db.test.count({"z": -10})) db.test.update_one({"z": -9}, {"$inc": {"z": 9}}, bypass_document_validation=False) self.assertEqual(0, db.test.count({"z": -9})) self.assertEqual(1, db.test.count({"z": 0})) self.assertRaises(OperationFailure, db_w0.test.update_one, {"y": 1}, {"$inc": {"x": 1}}, bypass_document_validation=True) # Test update_many db.test.insert_many([{"z": i} for i in range(3, 101)]) db.test.insert_one({"y": 0}, bypass_document_validation=True) self.assertRaises(OperationFailure, db.test.update_many, {}, {"$inc": {"z": -100}}) self.assertEqual(100, db.test.count({"z": {"$gte": 0}})) self.assertEqual(0, db.test.count({"z": {"$lt": 0}})) self.assertEqual(0, db.test.count({"y": 0, "z": -100})) db.test.update_many({"z": {"$gte": 0}}, {"$inc": {"z": -100}}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"z": {"$gt": 0}})) self.assertEqual(100, db.test.count({"z": {"$lte": 0}})) db.test.update_many({"z": {"$gt": -50}}, {"$inc": {"z": 100}}, bypass_document_validation=False) self.assertEqual(50, db.test.count({"z": {"$gt": 0}})) self.assertEqual(50, db.test.count({"z": {"$lt": 0}})) db.test.insert_many([{"z": -i} for i in range(50)], bypass_document_validation=True) self.assertRaises(OperationFailure, db.test.update_many, {}, {"$inc": {"z": 1}}) self.assertEqual(100, db.test.count({"z": {"$lte": 0}})) self.assertEqual(50, db.test.count({"z": {"$gt": 1}})) db.test.update_many({"z": {"$gte": 0}}, {"$inc": {"z": -100}}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"z": {"$gt": 0}})) self.assertEqual(150, db.test.count({"z": {"$lte": 0}})) db.test.update_many({"z": {"$lte": 0}}, {"$inc": {"z": 100}}, bypass_document_validation=False) self.assertEqual(150, db.test.count({"z": {"$gte": 0}})) self.assertEqual(0, db.test.count({"z": {"$lt": 0}})) self.assertRaises(OperationFailure, db_w0.test.update_many, {"y": 1}, {"$inc": {"x": 1}}, bypass_document_validation=True) @client_context.require_version_min(3, 1, 9, -1) def test_bypass_document_validation_bulk_write(self): db = self.db db.test.drop() db.create_collection("test", validator={"a": {"$gte": 0}}) db_w0 = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=0)) ops = [InsertOne({"a": -10}), InsertOne({"a": -11}), InsertOne({"a": -12}), UpdateOne({"a": {"$lte": -10}}, {"$inc": {"a": 1}}), UpdateMany({"a": {"$lte": -10}}, {"$inc": {"a": 1}}), ReplaceOne({"a": {"$lte": -10}}, {"a": -1})] db.test.bulk_write(ops, bypass_document_validation=True) self.assertEqual(3, db.test.count()) self.assertEqual(1, db.test.count({"a": -11})) self.assertEqual(1, db.test.count({"a": -1})) self.assertEqual(1, db.test.count({"a": -9})) # Assert that the operations would fail without bypass_doc_val for op in ops: self.assertRaises(BulkWriteError, db.test.bulk_write, [op]) self.assertRaises(OperationFailure, db_w0.test.bulk_write, ops, bypass_document_validation=True) def test_find_by_default_dct(self): db = self.db db.test.insert_one({'foo': 'bar'}) dct = defaultdict(dict, [('foo', 'bar')]) self.assertIsNotNone(db.test.find_one(dct)) self.assertEqual(dct, defaultdict(dict, [('foo', 'bar')])) def test_find_w_fields(self): db = self.db db.test.delete_many({}) db.test.insert_one({"x": 1, "mike": "awesome", "extra thing": "abcdefghijklmnopqrstuvwxyz"}) self.assertEqual(1, db.test.count()) doc = next(db.test.find({})) self.assertTrue("x" in doc) doc = next(db.test.find({})) self.assertTrue("mike" in doc) doc = next(db.test.find({})) self.assertTrue("extra thing" in doc) doc = next(db.test.find({}, ["x", "mike"])) self.assertTrue("x" in doc) doc = next(db.test.find({}, ["x", "mike"])) self.assertTrue("mike" in doc) doc = next(db.test.find({}, ["x", "mike"])) self.assertFalse("extra thing" in doc) doc = next(db.test.find({}, ["mike"])) self.assertFalse("x" in doc) doc = next(db.test.find({}, ["mike"])) self.assertTrue("mike" in doc) doc = next(db.test.find({}, ["mike"])) self.assertFalse("extra thing" in doc) def test_fields_specifier_as_dict(self): db = self.db db.test.delete_many({}) db.test.insert_one({"x": [1, 2, 3], "mike": "awesome"}) self.assertEqual([1, 2, 3], db.test.find_one()["x"]) self.assertEqual([2, 3], db.test.find_one( projection={"x": {"$slice": -2}})["x"]) self.assertTrue("x" not in db.test.find_one(projection={"x": 0})) self.assertTrue("mike" in db.test.find_one(projection={"x": 0})) def test_find_w_regex(self): db = self.db db.test.delete_many({}) db.test.insert_one({"x": "hello_world"}) db.test.insert_one({"x": "hello_mike"}) db.test.insert_one({"x": "hello_mikey"}) db.test.insert_one({"x": "hello_test"}) self.assertEqual(db.test.find().count(), 4) self.assertEqual(db.test.find({"x": re.compile("^hello.*")}).count(), 4) self.assertEqual(db.test.find({"x": re.compile("ello")}).count(), 4) self.assertEqual(db.test.find({"x": re.compile("^hello$")}).count(), 0) self.assertEqual(db.test.find({"x": re.compile("^hello_mi.*$")}).count(), 2) def test_id_can_be_anything(self): db = self.db db.test.delete_many({}) auto_id = {"hello": "world"} db.test.insert_one(auto_id) self.assertTrue(isinstance(auto_id["_id"], ObjectId)) numeric = {"_id": 240, "hello": "world"} db.test.insert_one(numeric) self.assertEqual(numeric["_id"], 240) obj = {"_id": numeric, "hello": "world"} db.test.insert_one(obj) self.assertEqual(obj["_id"], numeric) for x in db.test.find(): self.assertEqual(x["hello"], u"world") self.assertTrue("_id" in x) def test_invalid_key_names(self): db = self.db db.test.drop() db.test.insert_one({"hello": "world"}) db.test.insert_one({"hello": {"hello": "world"}}) self.assertRaises(InvalidDocument, db.test.insert_one, {"$hello": "world"}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello": {"$hello": "world"}}) db.test.insert_one({"he$llo": "world"}) db.test.insert_one({"hello": {"hello$": "world"}}) self.assertRaises(InvalidDocument, db.test.insert_one, {".hello": "world"}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello": {".hello": "world"}}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello.": "world"}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello": {"hello.": "world"}}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hel.lo": "world"}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello": {"hel.lo": "world"}}) def test_unique_index(self): db = self.db db.drop_collection("test") db.test.create_index("hello") # No error. db.test.insert_one({"hello": "world"}) db.test.insert_one({"hello": "world"}) db.drop_collection("test") db.test.create_index("hello", unique=True) with self.assertRaises(DuplicateKeyError): db.test.insert_one({"hello": "world"}) db.test.insert_one({"hello": "world"}) def test_duplicate_key_error(self): db = self.db db.drop_collection("test") db.test.create_index("x", unique=True) db.test.insert_one({"_id": 1, "x": 1}) with self.assertRaises(DuplicateKeyError) as context: db.test.insert_one({"x": 1}) self.assertIsNotNone(context.exception.details) with self.assertRaises(DuplicateKeyError) as context: db.test.insert_one({"x": 1}) self.assertIsNotNone(context.exception.details) self.assertEqual(1, db.test.count()) def test_write_error_text_handling(self): db = self.db db.drop_collection("test") db.test.create_index("text", unique=True) # Test workaround for SERVER-24007 data = (b'a\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83' b'\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83\xe2\x98\x83') text = utf_8_decode(data, None, True) db.test.insert_one({"text": text}) # Should raise DuplicateKeyError, not InvalidBSON self.assertRaises(DuplicateKeyError, db.test.insert_one, {"text": text}) self.assertRaises(DuplicateKeyError, db.test.insert, {"text": text}) self.assertRaises(DuplicateKeyError, db.test.insert, [{"text": text}]) self.assertRaises(DuplicateKeyError, db.test.replace_one, {"_id": ObjectId()}, {"text": text}, upsert=True) self.assertRaises(DuplicateKeyError, db.test.update, {"_id": ObjectId()}, {"text": text}, upsert=True) # Should raise BulkWriteError, not InvalidBSON self.assertRaises(BulkWriteError, db.test.insert_many, [{"text": text}]) def test_wtimeout(self): # Ensure setting wtimeout doesn't disable write concern altogether. # See SERVER-12596. collection = self.db.test collection.drop() collection.insert_one({'_id': 1}) coll = collection.with_options( write_concern=WriteConcern(w=1, wtimeout=1000)) self.assertRaises(DuplicateKeyError, coll.insert_one, {'_id': 1}) coll = collection.with_options( write_concern=WriteConcern(wtimeout=1000)) self.assertRaises(DuplicateKeyError, coll.insert_one, {'_id': 1}) def test_error_code(self): try: self.db.test.update_many({}, {"$thismodifierdoesntexist": 1}) except OperationFailure as exc: self.assertTrue(exc.code in (9, 10147, 16840, 17009)) # Just check that we set the error document. Fields # vary by MongoDB version. self.assertTrue(exc.details is not None) else: self.fail("OperationFailure was not raised") def test_index_on_subfield(self): db = self.db db.drop_collection("test") db.test.insert_one({"hello": {"a": 4, "b": 5}}) db.test.insert_one({"hello": {"a": 7, "b": 2}}) db.test.insert_one({"hello": {"a": 4, "b": 10}}) db.drop_collection("test") db.test.create_index("hello.a", unique=True) db.test.insert_one({"hello": {"a": 4, "b": 5}}) db.test.insert_one({"hello": {"a": 7, "b": 2}}) self.assertRaises(DuplicateKeyError, db.test.insert_one, {"hello": {"a": 4, "b": 10}}) def test_replace_one(self): db = self.db db.drop_collection("test") self.assertRaises(ValueError, lambda: db.test.replace_one({}, {"$set": {"x": 1}})) id1 = db.test.insert_one({"x": 1}).inserted_id result = db.test.replace_one({"x": 1}, {"y": 1}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(1, db.test.count({"y": 1})) self.assertEqual(0, db.test.count({"x": 1})) self.assertEqual(db.test.find_one(id1)["y"], 1) replacement = RawBSONDocument(bson.BSON.encode({"_id": id1, "z": 1})) result = db.test.replace_one({"y": 1}, replacement, True) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(1, db.test.count({"z": 1})) self.assertEqual(0, db.test.count({"y": 1})) self.assertEqual(db.test.find_one(id1)["z"], 1) result = db.test.replace_one({"x": 2}, {"y": 2}, True) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(0, result.matched_count) self.assertTrue(result.modified_count in (None, 0)) self.assertTrue(isinstance(result.upserted_id, ObjectId)) self.assertTrue(result.acknowledged) self.assertEqual(1, db.test.count({"y": 2})) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) result = db.test.replace_one({"x": 0}, {"y": 0}) self.assertTrue(isinstance(result, UpdateResult)) self.assertRaises(InvalidOperation, lambda: result.matched_count) self.assertRaises(InvalidOperation, lambda: result.modified_count) self.assertRaises(InvalidOperation, lambda: result.upserted_id) self.assertFalse(result.acknowledged) def test_update_one(self): db = self.db db.drop_collection("test") self.assertRaises(ValueError, lambda: db.test.update_one({}, {"x": 1})) id1 = db.test.insert_one({"x": 5}).inserted_id result = db.test.update_one({}, {"$inc": {"x": 1}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(db.test.find_one(id1)["x"], 6) id2 = db.test.insert_one({"x": 1}).inserted_id result = db.test.update_one({"x": 6}, {"$inc": {"x": 1}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(db.test.find_one(id1)["x"], 7) self.assertEqual(db.test.find_one(id2)["x"], 1) result = db.test.update_one({"x": 2}, {"$set": {"y": 1}}, True) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(0, result.matched_count) self.assertTrue(result.modified_count in (None, 0)) self.assertTrue(isinstance(result.upserted_id, ObjectId)) self.assertTrue(result.acknowledged) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) result = db.test.update_one({"x": 0}, {"$inc": {"x": 1}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertRaises(InvalidOperation, lambda: result.matched_count) self.assertRaises(InvalidOperation, lambda: result.modified_count) self.assertRaises(InvalidOperation, lambda: result.upserted_id) self.assertFalse(result.acknowledged) def test_update_many(self): db = self.db db.drop_collection("test") self.assertRaises(ValueError, lambda: db.test.update_many({}, {"x": 1})) db.test.insert_one({"x": 4, "y": 3}) db.test.insert_one({"x": 5, "y": 5}) db.test.insert_one({"x": 4, "y": 4}) result = db.test.update_many({"x": 4}, {"$set": {"y": 5}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(2, result.matched_count) self.assertTrue(result.modified_count in (None, 2)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(3, db.test.count({"y": 5})) result = db.test.update_many({"x": 5}, {"$set": {"y": 6}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(1, db.test.count({"y": 6})) result = db.test.update_many({"x": 2}, {"$set": {"y": 1}}, True) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(0, result.matched_count) self.assertTrue(result.modified_count in (None, 0)) self.assertTrue(isinstance(result.upserted_id, ObjectId)) self.assertTrue(result.acknowledged) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) result = db.test.update_many({"x": 0}, {"$inc": {"x": 1}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertRaises(InvalidOperation, lambda: result.matched_count) self.assertRaises(InvalidOperation, lambda: result.modified_count) self.assertRaises(InvalidOperation, lambda: result.upserted_id) self.assertFalse(result.acknowledged) # MongoDB >= 3.5.8 allows dotted fields in updates @client_context.require_version_max(3, 5, 7) def test_update_with_invalid_keys(self): self.db.drop_collection("test") self.assertTrue(self.db.test.insert_one({"hello": "world"})) doc = self.db.test.find_one() doc['a.b'] = 'c' # Replace self.assertRaises(OperationFailure, self.db.test.replace_one, {"hello": "world"}, doc) # Upsert self.assertRaises(OperationFailure, self.db.test.replace_one, {"foo": "bar"}, doc, upsert=True) # Check that the last two ops didn't actually modify anything self.assertTrue('a.b' not in self.db.test.find_one()) def test_update_check_keys(self): self.db.drop_collection("test") self.assertTrue(self.db.test.insert_one({"hello": "world"})) # Modify shouldn't check keys... self.assertTrue(self.db.test.update_one({"hello": "world"}, {"$set": {"foo.bar": "baz"}}, upsert=True)) # I know this seems like testing the server but I'd like to be notified # by CI if the server's behavior changes here. doc = SON([("$set", {"foo.bar": "bim"}), ("hello", "world")]) self.assertRaises(OperationFailure, self.db.test.update_one, {"hello": "world"}, doc, upsert=True) # This is going to cause keys to be checked and raise InvalidDocument. # That's OK assuming the server's behavior in the previous assert # doesn't change. If the behavior changes checking the first key for # '$' in update won't be good enough anymore. doc = SON([("hello", "world"), ("$set", {"foo.bar": "bim"})]) self.assertRaises(OperationFailure, self.db.test.replace_one, {"hello": "world"}, doc, upsert=True) # Replace with empty document self.assertNotEqual(0, self.db.test.replace_one( {"hello": "world"}, {}).matched_count) def test_acknowledged_delete(self): db = self.db db.drop_collection("test") db.create_collection("test", capped=True, size=1000) db.test.insert_one({"x": 1}) self.assertEqual(1, db.test.count()) # Can't remove from capped collection. self.assertRaises(OperationFailure, db.test.delete_one, {"x": 1}) db.drop_collection("test") db.test.insert_one({"x": 1}) db.test.insert_one({"x": 1}) self.assertEqual(2, db.test.delete_many({}).deleted_count) self.assertEqual(0, db.test.delete_many({}).deleted_count) def test_manual_last_error(self): coll = self.db.get_collection("test", write_concern=WriteConcern(w=0)) coll.insert_one({"x": 1}) self.db.command("getlasterror", w=1, wtimeout=1) def test_count(self): db = self.db db.drop_collection("test") self.assertEqual(db.test.count(), 0) db.test.insert_many([{}, {}]) self.assertEqual(db.test.count(), 2) db.test.insert_many([{'foo': 'bar'}, {'foo': 'baz'}]) self.assertEqual(db.test.find({'foo': 'bar'}).count(), 1) self.assertEqual(db.test.count({'foo': 'bar'}), 1) self.assertEqual(db.test.find({'foo': re.compile(r'ba.*')}).count(), 2) self.assertEqual( db.test.count({'foo': re.compile(r'ba.*')}), 2) def test_aggregate(self): db = self.db db.drop_collection("test") db.test.insert_one({'foo': [1, 2]}) self.assertRaises(TypeError, db.test.aggregate, "wow") pipeline = {"$project": {"_id": False, "foo": True}} # MongoDB 3.5.1+ requires either the 'cursor' or 'explain' options. if client_context.version.at_least(3, 5, 1): result = db.test.aggregate([pipeline]) else: result = db.test.aggregate([pipeline], useCursor=False) self.assertTrue(isinstance(result, CommandCursor)) self.assertEqual([{'foo': [1, 2]}], list(result)) # Test write concern. out_pipeline = [pipeline, {'$out': 'output-collection'}] with self.write_concern_collection() as coll: coll.aggregate(out_pipeline) def test_aggregate_raw_bson(self): db = self.db db.drop_collection("test") db.test.insert_one({'foo': [1, 2]}) self.assertRaises(TypeError, db.test.aggregate, "wow") pipeline = {"$project": {"_id": False, "foo": True}} coll = db.get_collection( 'test', codec_options=CodecOptions(document_class=RawBSONDocument)) # MongoDB 3.5.1+ requires either the 'cursor' or 'explain' options. if client_context.version.at_least(3, 5, 1): result = coll.aggregate([pipeline]) else: result = coll.aggregate([pipeline], useCursor=False) self.assertTrue(isinstance(result, CommandCursor)) first_result = next(result) self.assertIsInstance(first_result, RawBSONDocument) self.assertEqual([1, 2], list(first_result['foo'])) def test_aggregation_cursor_validation(self): db = self.db projection = {'$project': {'_id': '$_id'}} cursor = db.test.aggregate([projection], cursor={}) self.assertTrue(isinstance(cursor, CommandCursor)) cursor = db.test.aggregate([projection], useCursor=True) self.assertTrue(isinstance(cursor, CommandCursor)) def test_aggregation_cursor(self): db = self.db if client_context.has_secondaries: # Test that getMore messages are sent to the right server. db = self.client.get_database( db.name, read_preference=ReadPreference.SECONDARY, write_concern=WriteConcern(w=self.w)) for collection_size in (10, 1000): db.drop_collection("test") db.test.insert_many([{'_id': i} for i in range(collection_size)]) expected_sum = sum(range(collection_size)) # Use batchSize to ensure multiple getMore messages cursor = db.test.aggregate( [{'$project': {'_id': '$_id'}}], batchSize=5) self.assertEqual( expected_sum, sum(doc['_id'] for doc in cursor)) # Test that batchSize is handled properly. cursor = db.test.aggregate([], batchSize=5) self.assertEqual(5, len(cursor._CommandCursor__data)) # Force a getMore cursor._CommandCursor__data.clear() next(cursor) # batchSize - 1 self.assertEqual(4, len(cursor._CommandCursor__data)) # Exhaust the cursor. There shouldn't be any errors. for doc in cursor: pass def test_aggregation_cursor_alive(self): self.db.test.delete_many({}) self.db.test.insert_many([{} for _ in range(3)]) self.addCleanup(self.db.test.delete_many, {}) cursor = self.db.test.aggregate(pipeline=[], cursor={'batchSize': 2}) n = 0 while True: cursor.next() n += 1 if 3 == n: self.assertFalse(cursor.alive) break self.assertTrue(cursor.alive) @client_context.require_no_mongos def test_parallel_scan(self): db = self.db db.drop_collection("test") if client_context.has_secondaries: # Test that getMore messages are sent to the right server. db = self.client.get_database( db.name, read_preference=ReadPreference.SECONDARY, write_concern=WriteConcern(w=self.w)) coll = db.test coll.insert_many([{'_id': i} for i in range(8000)]) docs = [] threads = [threading.Thread(target=docs.extend, args=(cursor,)) for cursor in coll.parallel_scan(3)] for t in threads: t.start() for t in threads: t.join() self.assertEqual( set(range(8000)), set(doc['_id'] for doc in docs)) @client_context.require_no_mongos @client_context.require_version_min(3, 3, 10) @client_context.require_test_commands def test_parallel_scan_max_time_ms(self): self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") try: self.assertRaises(ExecutionTimeout, self.db.test.parallel_scan, 3, maxTimeMS=1) finally: self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") def test_large_limit(self): db = self.db db.drop_collection("test_large_limit") db.test_large_limit.create_index([('x', 1)]) my_str = "mongomongo" * 1000 for i in range(2000): doc = {"x": i, "y": my_str} db.test_large_limit.insert_one(doc) i = 0 y = 0 for doc in db.test_large_limit.find(limit=1900).sort([('x', 1)]): i += 1 y += doc["x"] self.assertEqual(1900, i) self.assertEqual((1900 * 1899) / 2, y) def test_find_kwargs(self): db = self.db db.drop_collection("test") for i in range(10): db.test.insert_one({"x": i}) self.assertEqual(10, db.test.count()) total = 0 for x in db.test.find({}, skip=4, limit=2): total += x["x"] self.assertEqual(9, total) def test_rename(self): db = self.db db.drop_collection("test") db.drop_collection("foo") self.assertRaises(TypeError, db.test.rename, 5) self.assertRaises(InvalidName, db.test.rename, "") self.assertRaises(InvalidName, db.test.rename, "te$t") self.assertRaises(InvalidName, db.test.rename, ".test") self.assertRaises(InvalidName, db.test.rename, "test.") self.assertRaises(InvalidName, db.test.rename, "tes..t") self.assertEqual(0, db.test.count()) self.assertEqual(0, db.foo.count()) for i in range(10): db.test.insert_one({"x": i}) self.assertEqual(10, db.test.count()) db.test.rename("foo") self.assertEqual(0, db.test.count()) self.assertEqual(10, db.foo.count()) x = 0 for doc in db.foo.find(): self.assertEqual(x, doc["x"]) x += 1 db.test.insert_one({}) self.assertRaises(OperationFailure, db.foo.rename, "test") db.foo.rename("test", dropTarget=True) with self.write_concern_collection() as coll: coll.rename('foo') def test_find_one(self): db = self.db db.drop_collection("test") _id = db.test.insert_one({"hello": "world", "foo": "bar"}).inserted_id self.assertEqual("world", db.test.find_one()["hello"]) self.assertEqual(db.test.find_one(_id), db.test.find_one()) self.assertEqual(db.test.find_one(None), db.test.find_one()) self.assertEqual(db.test.find_one({}), db.test.find_one()) self.assertEqual(db.test.find_one({"hello": "world"}), db.test.find_one()) self.assertTrue("hello" in db.test.find_one(projection=["hello"])) self.assertTrue("hello" not in db.test.find_one(projection=["foo"])) self.assertTrue("hello" in db.test.find_one(projection=("hello",))) self.assertTrue("hello" not in db.test.find_one(projection=("foo",))) self.assertTrue("hello" in db.test.find_one(projection=set(["hello"]))) self.assertTrue("hello" not in db.test.find_one(projection=set(["foo"]))) self.assertTrue("hello" in db.test.find_one(projection=frozenset(["hello"]))) self.assertTrue("hello" not in db.test.find_one(projection=frozenset(["foo"]))) self.assertEqual(["_id"], list(db.test.find_one(projection=[]))) self.assertEqual(None, db.test.find_one({"hello": "foo"})) self.assertEqual(None, db.test.find_one(ObjectId())) def test_find_one_non_objectid(self): db = self.db db.drop_collection("test") db.test.insert_one({"_id": 5}) self.assertTrue(db.test.find_one(5)) self.assertFalse(db.test.find_one(6)) def test_find_one_with_find_args(self): db = self.db db.drop_collection("test") db.test.insert_many([{"x": i} for i in range(1, 4)]) self.assertEqual(1, db.test.find_one()["x"]) self.assertEqual(2, db.test.find_one(skip=1, limit=2)["x"]) def test_find_with_sort(self): db = self.db db.drop_collection("test") db.test.insert_many([{"x": 2}, {"x": 1}, {"x": 3}]) self.assertEqual(2, db.test.find_one()["x"]) self.assertEqual(1, db.test.find_one(sort=[("x", 1)])["x"]) self.assertEqual(3, db.test.find_one(sort=[("x", -1)])["x"]) def to_list(things): return [thing["x"] for thing in things] self.assertEqual([2, 1, 3], to_list(db.test.find())) self.assertEqual([1, 2, 3], to_list(db.test.find(sort=[("x", 1)]))) self.assertEqual([3, 2, 1], to_list(db.test.find(sort=[("x", -1)]))) self.assertRaises(TypeError, db.test.find, sort=5) self.assertRaises(TypeError, db.test.find, sort="hello") self.assertRaises(ValueError, db.test.find, sort=["hello", 1]) # TODO doesn't actually test functionality, just that it doesn't blow up def test_cursor_timeout(self): list(self.db.test.find(no_cursor_timeout=True)) list(self.db.test.find(no_cursor_timeout=False)) def test_exhaust(self): if is_mongos(self.db.client): self.assertRaises(InvalidOperation, self.db.test.find, cursor_type=CursorType.EXHAUST) return # Limit is incompatible with exhaust. self.assertRaises(InvalidOperation, self.db.test.find, cursor_type=CursorType.EXHAUST, limit=5) cur = self.db.test.find(cursor_type=CursorType.EXHAUST) self.assertRaises(InvalidOperation, cur.limit, 5) cur = self.db.test.find(limit=5) self.assertRaises(InvalidOperation, cur.add_option, 64) cur = self.db.test.find() cur.add_option(64) self.assertRaises(InvalidOperation, cur.limit, 5) self.db.drop_collection("test") # Insert enough documents to require more than one batch self.db.test.insert_many([{'i': i} for i in range(150)]) client = rs_or_single_client(maxPoolSize=1) socks = get_pool(client).sockets # Make sure the socket is returned after exhaustion. cur = client[self.db.name].test.find(cursor_type=CursorType.EXHAUST) next(cur) self.assertEqual(0, len(socks)) for _ in cur: pass self.assertEqual(1, len(socks)) # Same as previous but don't call next() for _ in client[self.db.name].test.find(cursor_type=CursorType.EXHAUST): pass self.assertEqual(1, len(socks)) # If the Cursor instance is discarded before being # completely iterated we have to close and # discard the socket. cur = client[self.db.name].test.find(cursor_type=CursorType.EXHAUST) next(cur) self.assertEqual(0, len(socks)) if sys.platform.startswith('java') or 'PyPy' in sys.version: # Don't wait for GC or use gc.collect(), it's unreliable. cur.close() cur = None # The socket should be discarded. self.assertEqual(0, len(socks)) def test_distinct(self): self.db.drop_collection("test") test = self.db.test test.insert_many([{"a": 1}, {"a": 2}, {"a": 2}, {"a": 2}, {"a": 3}]) distinct = test.distinct("a") distinct.sort() self.assertEqual([1, 2, 3], distinct) distinct = test.find({'a': {'$gt': 1}}).distinct("a") distinct.sort() self.assertEqual([2, 3], distinct) distinct = test.distinct('a', {'a': {'$gt': 1}}) distinct.sort() self.assertEqual([2, 3], distinct) self.db.drop_collection("test") test.insert_one({"a": {"b": "a"}, "c": 12}) test.insert_one({"a": {"b": "b"}, "c": 12}) test.insert_one({"a": {"b": "c"}, "c": 12}) test.insert_one({"a": {"b": "c"}, "c": 12}) distinct = test.distinct("a.b") distinct.sort() self.assertEqual(["a", "b", "c"], distinct) def test_query_on_query_field(self): self.db.drop_collection("test") self.db.test.insert_one({"query": "foo"}) self.db.test.insert_one({"bar": "foo"}) self.assertEqual(1, self.db.test.find({"query": {"$ne": None}}).count()) self.assertEqual(1, len(list(self.db.test.find({"query": {"$ne": None}}))) ) def test_min_query(self): self.db.drop_collection("test") self.db.test.insert_many([{"x": 1}, {"x": 2}]) self.db.test.create_index("x") self.assertEqual(1, len(list(self.db.test.find({"$min": {"x": 2}, "$query": {}})))) self.assertEqual(2, self.db.test.find({"$min": {"x": 2}, "$query": {}})[0]["x"]) def test_numerous_inserts(self): # Ensure we don't exceed server's 1000-document batch size limit. self.db.test.drop() n_docs = 2100 self.db.test.insert_many([{} for _ in range(n_docs)]) self.assertEqual(n_docs, self.db.test.count()) self.db.test.drop() def test_map_reduce(self): db = self.db db.drop_collection("test") db.test.insert_one({"id": 1, "tags": ["dog", "cat"]}) db.test.insert_one({"id": 2, "tags": ["cat"]}) db.test.insert_one({"id": 3, "tags": ["mouse", "cat", "dog"]}) db.test.insert_one({"id": 4, "tags": []}) map = Code("function () {" " this.tags.forEach(function(z) {" " emit(z, 1);" " });" "}") reduce = Code("function (key, values) {" " var total = 0;" " for (var i = 0; i < values.length; i++) {" " total += values[i];" " }" " return total;" "}") result = db.test.map_reduce(map, reduce, out='mrunittests') self.assertEqual(3, result.find_one({"_id": "cat"})["value"]) self.assertEqual(2, result.find_one({"_id": "dog"})["value"]) self.assertEqual(1, result.find_one({"_id": "mouse"})["value"]) db.test.insert_one({"id": 5, "tags": ["hampster"]}) result = db.test.map_reduce(map, reduce, out='mrunittests') self.assertEqual(1, result.find_one({"_id": "hampster"})["value"]) db.test.delete_one({"id": 5}) result = db.test.map_reduce(map, reduce, out={'merge': 'mrunittests'}) self.assertEqual(3, result.find_one({"_id": "cat"})["value"]) self.assertEqual(1, result.find_one({"_id": "hampster"})["value"]) result = db.test.map_reduce(map, reduce, out={'reduce': 'mrunittests'}) self.assertEqual(6, result.find_one({"_id": "cat"})["value"]) self.assertEqual(4, result.find_one({"_id": "dog"})["value"]) self.assertEqual(2, result.find_one({"_id": "mouse"})["value"]) self.assertEqual(1, result.find_one({"_id": "hampster"})["value"]) result = db.test.map_reduce( map, reduce, out={'replace': 'mrunittests'} ) self.assertEqual(3, result.find_one({"_id": "cat"})["value"]) self.assertEqual(2, result.find_one({"_id": "dog"})["value"]) self.assertEqual(1, result.find_one({"_id": "mouse"})["value"]) result = db.test.map_reduce(map, reduce, out=SON([('replace', 'mrunittests'), ('db', 'mrtestdb') ])) self.assertEqual(3, result.find_one({"_id": "cat"})["value"]) self.assertEqual(2, result.find_one({"_id": "dog"})["value"]) self.assertEqual(1, result.find_one({"_id": "mouse"})["value"]) self.client.drop_database('mrtestdb') full_result = db.test.map_reduce(map, reduce, out='mrunittests', full_response=True) self.assertEqual(6, full_result["counts"]["emit"]) result = db.test.map_reduce(map, reduce, out='mrunittests', limit=2) self.assertEqual(2, result.find_one({"_id": "cat"})["value"]) self.assertEqual(1, result.find_one({"_id": "dog"})["value"]) self.assertEqual(None, result.find_one({"_id": "mouse"})) result = db.test.map_reduce(map, reduce, out={'inline': 1}) self.assertTrue(isinstance(result, dict)) self.assertTrue('results' in result) self.assertTrue(result['results'][1]["_id"] in ("cat", "dog", "mouse")) result = db.test.inline_map_reduce(map, reduce) self.assertTrue(isinstance(result, list)) self.assertEqual(3, len(result)) self.assertTrue(result[1]["_id"] in ("cat", "dog", "mouse")) full_result = db.test.inline_map_reduce(map, reduce, full_response=True) self.assertEqual(6, full_result["counts"]["emit"]) with self.write_concern_collection() as coll: coll.map_reduce(map, reduce, 'output') def test_messages_with_unicode_collection_names(self): db = self.db db[u"Employés"].insert_one({"x": 1}) db[u"Employés"].replace_one({"x": 1}, {"x": 2}) db[u"Employés"].delete_many({}) db[u"Employés"].find_one() list(db[u"Employés"].find()) def test_drop_indexes_non_existent(self): self.db.drop_collection("test") self.db.test.drop_indexes() # This is really a bson test but easier to just reproduce it here... # (Shame on me) def test_bad_encode(self): c = self.db.test c.drop() self.assertRaises(InvalidDocument, c.insert_one, {"x": c}) class BadGetAttr(dict): def __getattr__(self, name): pass bad = BadGetAttr([('foo', 'bar')]) c.insert_one({'bad': bad}) self.assertEqual('bar', c.find_one()['bad']['foo']) @client_context.require_version_max(3, 5, 5) def test_array_filters_unsupported(self): c = self.db.test with self.assertRaises(ConfigurationError): c.update_one( {}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]) with self.assertRaises(ConfigurationError): c.update_many( {}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]) with self.assertRaises(ConfigurationError): c.find_one_and_update( {}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]) def test_array_filters_validation(self): # array_filters must be a list. c = self.db.test with self.assertRaises(TypeError): c.update_one({}, {'$set': {'a': 1}}, array_filters={}) with self.assertRaises(TypeError): c.update_many({}, {'$set': {'a': 1}}, array_filters={}) with self.assertRaises(TypeError): c.find_one_and_update({}, {'$set': {'a': 1}}, array_filters={}) def test_array_filters_unacknowledged(self): c_w0 = self.db.test.with_options(write_concern=WriteConcern(w=0)) with self.assertRaises(ConfigurationError): c_w0.update_one({}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]) with self.assertRaises(ConfigurationError): c_w0.update_many({}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]) with self.assertRaises(ConfigurationError): c_w0.find_one_and_update({}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]) def test_find_one_and(self): c = self.db.test c.drop() c.insert_one({'_id': 1, 'i': 1}) self.assertEqual({'_id': 1, 'i': 1}, c.find_one_and_update({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual({'_id': 1, 'i': 3}, c.find_one_and_update( {'_id': 1}, {'$inc': {'i': 1}}, return_document=ReturnDocument.AFTER)) self.assertEqual({'_id': 1, 'i': 3}, c.find_one_and_delete({'_id': 1})) self.assertEqual(None, c.find_one({'_id': 1})) self.assertEqual(None, c.find_one_and_update({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual({'_id': 1, 'i': 1}, c.find_one_and_update( {'_id': 1}, {'$inc': {'i': 1}}, return_document=ReturnDocument.AFTER, upsert=True)) self.assertEqual({'_id': 1, 'i': 2}, c.find_one_and_update( {'_id': 1}, {'$inc': {'i': 1}}, return_document=ReturnDocument.AFTER)) self.assertEqual({'_id': 1, 'i': 3}, c.find_one_and_replace( {'_id': 1}, {'i': 3, 'j': 1}, projection=['i'], return_document=ReturnDocument.AFTER)) self.assertEqual({'i': 4}, c.find_one_and_update( {'_id': 1}, {'$inc': {'i': 1}}, projection={'i': 1, '_id': 0}, return_document=ReturnDocument.AFTER)) c.drop() for j in range(5): c.insert_one({'j': j, 'i': 0}) sort = [('j', DESCENDING)] self.assertEqual(4, c.find_one_and_update({}, {'$inc': {'i': 1}}, sort=sort)['j']) def test_find_one_and_write_concern(self): listener = EventListener() saved_listeners = monitoring._LISTENERS monitoring._LISTENERS = monitoring._Listeners([], [], [], []) db = single_client(event_listeners=[listener])[self.db.name] # non-default WriteConcern. c_w0 = db.get_collection( 'test', write_concern=WriteConcern(w=0)) # default WriteConcern. c_default = db.get_collection('test', write_concern=WriteConcern()) results = listener.results # Authenticate the client and throw out auth commands from the listener. db.command('ismaster') results.clear() try: if client_context.version.at_least(3, 1, 9, -1): c_w0.find_and_modify( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertEqual( {'w': 0}, results['started'][0].command['writeConcern']) results.clear() c_w0.find_one_and_update( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertEqual( {'w': 0}, results['started'][0].command['writeConcern']) results.clear() c_w0.find_one_and_replace({'_id': 1}, {'foo': 'bar'}) self.assertEqual( {'w': 0}, results['started'][0].command['writeConcern']) results.clear() c_w0.find_one_and_delete({'_id': 1}) self.assertEqual( {'w': 0}, results['started'][0].command['writeConcern']) results.clear() # Test write concern errors. if client_context.is_rs: c_wc_error = db.get_collection( 'test', write_concern=WriteConcern( w=len(client_context.nodes) + 1)) self.assertRaises( WriteConcernError, c_wc_error.find_and_modify, {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertRaises( WriteConcernError, c_wc_error.find_one_and_update, {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertRaises( WriteConcernError, c_wc_error.find_one_and_replace, {'w': 0}, results['started'][0].command['writeConcern']) self.assertRaises( WriteConcernError, c_wc_error.find_one_and_delete, {'w': 0}, results['started'][0].command['writeConcern']) results.clear() else: c_w0.find_and_modify( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_w0.find_one_and_update( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_w0.find_one_and_replace({'_id': 1}, {'foo': 'bar'}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_w0.find_one_and_delete({'_id': 1}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_default.find_and_modify({'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_default.find_one_and_update({'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_default.find_one_and_replace({'_id': 1}, {'foo': 'bar'}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_default.find_one_and_delete({'_id': 1}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() finally: monitoring._LISTENERS = saved_listeners def test_find_with_nested(self): c = self.db.test c.drop() c.insert_many([{'i': i} for i in range(5)]) # [0, 1, 2, 3, 4] self.assertEqual( [2], [i['i'] for i in c.find({ '$and': [ { # This clause gives us [1,2,4] '$or': [ {'i': {'$lte': 2}}, {'i': {'$gt': 3}}, ], }, { # This clause gives us [2,3] '$or': [ {'i': 2}, {'i': 3}, ] }, ] })] ) self.assertEqual( [0, 1, 2], [i['i'] for i in c.find({ '$or': [ { # This clause gives us [2] '$and': [ {'i': {'$gte': 2}}, {'i': {'$lt': 3}}, ], }, { # This clause gives us [0,1] '$and': [ {'i': {'$gt': -100}}, {'i': {'$lt': 2}}, ] }, ] })] ) def test_find_regex(self): c = self.db.test c.drop() c.insert_one({'r': re.compile('.*')}) self.assertTrue(isinstance(c.find_one()['r'], Regex)) for doc in c.find(): self.assertTrue(isinstance(doc['r'], Regex)) def test_find_command_generation(self): cmd = _gen_find_command('coll', {'$query': {'foo': 1}, '$dumb': 2}, None, 0, 0, 0, None, DEFAULT_READ_CONCERN, None) self.assertEqual( cmd.to_dict(), SON([('find', 'coll'), ('$dumb', 2), ('filter', {'foo': 1})]).to_dict()) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_dns.py0000644000076600000240000000727013245621354016622 0ustar shanestaff00000000000000# Copyright 2017 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Run the SRV support tests.""" import glob import json import os import sys sys.path[0:0] = [""] from pymongo.common import validate_read_preference_tags from pymongo.errors import ConfigurationError from pymongo.mongo_client import MongoClient from pymongo.uri_parser import parse_uri, split_hosts, _HAVE_DNSPYTHON from test import client_context, unittest from test.utils import wait_until _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'dns') _SSL_OPTS = client_context.ssl_client_options.copy() if client_context.ssl is True: # Our test certs don't support the SRV hosts used in these tests. _SSL_OPTS['ssl_match_hostname'] = False class TestDNS(unittest.TestCase): pass def create_test(test_case): @client_context.require_replica_set @client_context.require_ssl def run_test(self): if not _HAVE_DNSPYTHON: raise unittest.SkipTest("DNS tests require the dnspython module") uri = test_case['uri'] seeds = test_case['seeds'] hosts = test_case['hosts'] options = test_case.get('options') if seeds: seeds = split_hosts(','.join(seeds)) if hosts: hosts = frozenset(split_hosts(','.join(hosts))) if options: for key, value in options.items(): # Convert numbers / booleans to strings for comparison if isinstance(value, bool): options[key] = 'true' if value else 'false' elif isinstance(value, (int, float)): options[key] = str(value) if seeds: result = parse_uri(uri, validate=False) self.assertEqual(sorted(result['nodelist']), sorted(seeds)) if options: opts = result['options'] if 'readpreferencetags' in opts: rpts = validate_read_preference_tags( 'readPreferenceTags', opts.pop('readpreferencetags')) opts['readPreferenceTags'] = rpts self.assertEqual(result['options'], options) hostname = next(iter(client_context.client.nodes))[0] # The replica set members must be configured as 'localhost'. if hostname == 'localhost': client = MongoClient(uri, **_SSL_OPTS) # Force server selection client.admin.command('ismaster') wait_until( lambda: hosts == client.nodes, 'match test hosts to client nodes') else: try: parse_uri(uri) except (ConfigurationError, ValueError): pass else: self.fail("failed to raise an exception") return run_test def create_tests(): for filename in glob.glob(os.path.join(_TEST_PATH, '*.json')): test_suffix, _ = os.path.splitext(os.path.basename(filename)) with open(filename) as dns_test_file: test_method = create_test(json.load(dns_test_file)) setattr(TestDNS, 'test_' + test_suffix, test_method) create_tests() if __name__ == '__main__': unittest.main() pymongo-3.6.1/test/certificates/0000755000076600000240000000000013246104133017054 5ustar shanestaff00000000000000pymongo-3.6.1/test/certificates/server.pem0000644000076600000240000000564113245615620021102 0ustar shanestaff00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpgIBAAKCAQEAkqTc5ZPVWJHKlyYK5/hQqzS7ThE4vHHuKQD0ux+y7COMhjjR 2ILv6PIn/++7aON8Y/Ed1NBHP7Cq0Uo8eXR1st12Ehpnt132mWL8glPuKGgbaWPk pddKcBO5q7QsKj9/MKjR4eIIbw56ukaoDyye0Z3U0j8PIonuegGAk0Yqdd1B0BqA Wb1MphPvueY46Z3m4gUBl31HBabf44tYR9ma8ROYoMwypM9t48sXHNiqMwGeTSBA h6CfrTZXjC7Hpil2wfHS97p8ZT3EUgpegnOFvzXvyBqyXEQwr3tv5URpzupO14TP XUxLr+aDZMyqtt4tkgZjQvjKQVuUW0/qoQ0BUwIDAQABAoIBAQCGsW9EX7lClQbk NkkmMGrIFrNETYUzceIzP3GKvt71DuHwFxiIKhx6dpJO/r/A49JJahrgcj4/PeJo Qiux00qYc9oTXrWNM31h/g25F6ZU7urerqZBvbHdqACOufsnAxOseiPtulPMzhvk JSoQZgequbHVO56HNvIYlnCm1XgCsUSkSvl8PYnOARDIAWJuFiR4t/RDaOk+VRRp xTvmxJF1FsKeMuG/RzcCJEHYmhbOC+UWBAAailOTXm5ZOq/SWhq3I0EokaIQ0nSL vADXmp0NfHmCxB0Mc0a/R/ah3phIzCcHnc1PCMSobn7iabqP202FFi/R1XYmd+yc ngha8RxBAoGBAN3QA0kvDtTqomZmxh7ReiVy66ny/gDQy3f+fT7vE6pNBAp2/9AN 8mSBMa78SxJWpMm3CHmyAX5rb+dJ9mHyoxADeJcMbbwEt9aQqeMEWNpewhG1SbbY FD7CVlewUMv/GsteHudStyi07o5SKZU9tgWkZFPEVYWDTIuXqoD0QalbAoGBAKk+ 8sUpZJATuAkbhh/HO4Jcls/hNGwyClPpXm9wnupkB7KLwkffaWKwCXEfVB1/2ERB iwjtxFuzKZ93RgTI3aE0tZaScGDme7u3Z/Kj1kyOdgyVxyJD5qXKcp99AWLjTa+E VSne5/++yXOuum+bu4Rqeo6RRsd9cRcRekkYNpFpAoGBAJ6D0EVVpuDMCWDq430U sJjgpr/eUl1c2XuYWANIsaILxe6AmlIiFW5z0YC+9htV5g/tiNYHOwAQYXlFpxja YGPKRzyS+jzallJ1MaN18NWl0ET9bH+JrfYnxbKG40bVuV1KlwUzXIdvvefhmav4 3QLtN3GRpphye5pqucPMSrxvAoGBAJ42s1ourMynMmaJLWPtqqreBTnFzGzMhq0U vuwsetKgujVlwzPPHURdTiZK5CZDihecse2h3+rdXK6vIGx+nfkCPjLJKHbdX5QH CwxfroiYsLBpH+PfV/FqhKalhDM+TDQk4DwpHfYNE2OqVqzZB33s0C8QjfsVQDAR baCBGxnpAoGBALM6dqbPx2vHttYUrLQUyn1Tt+TAcCe1o/z/XlapGQfQAibkaCHG OuIpVZNh12MI375IkzW1BOITGx37x4LIR5f8DmdjZ5Ihw4nbAYSLRZh3NaPUwus7 en2tQtSymZziYzI+9EGmE6Sdaxj4FtM80wqF4j5pgcVqJMimgkJ23y7J -----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIDkDCCAnigAwIBAgIDBRdAMA0GCSqGSIb3DQEBBQUAMHUxGTAXBgNVBAMTEFB5 dGhvbiBEcml2ZXIgQ0ExEDAOBgNVBAsTB0RyaXZlcnMxEDAOBgNVBAoTB01vbmdv REIxEjAQBgNVBAcTCVBhbG8gQWx0bzETMBEGA1UECBMKQ2FsaWZvcm5pYTELMAkG A1UEBhMCVVMwHhcNMTcwMTMxMjExNzU4WhcNMzcwMTMxMjExNzU4WjBuMRIwEAYD VQQDEwlsb2NhbGhvc3QxEDAOBgNVBAsTB0RyaXZlcnMxEDAOBgNVBAoTB01vbmdv REIxEjAQBgNVBAcTCVBhbG8gQWx0bzETMBEGA1UECBMKQ2FsaWZvcm5pYTELMAkG A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCSpNzlk9VY kcqXJgrn+FCrNLtOETi8ce4pAPS7H7LsI4yGONHYgu/o8if/77to43xj8R3U0Ec/ sKrRSjx5dHWy3XYSGme3XfaZYvyCU+4oaBtpY+Sl10pwE7mrtCwqP38wqNHh4ghv Dnq6RqgPLJ7RndTSPw8iie56AYCTRip13UHQGoBZvUymE++55jjpnebiBQGXfUcF pt/ji1hH2ZrxE5igzDKkz23jyxcc2KozAZ5NIECHoJ+tNleMLsemKXbB8dL3unxl PcRSCl6Cc4W/Ne/IGrJcRDCve2/lRGnO6k7XhM9dTEuv5oNkzKq23i2SBmNC+MpB W5RbT+qhDQFTAgMBAAGjMDAuMCwGA1UdEQQlMCOCCWxvY2FsaG9zdIcEfwAAAYcQ AAAAAAAAAAAAAAAAAAAAATANBgkqhkiG9w0BAQUFAAOCAQEAU3Fa4NYVdE6ewPHR q3bsNnvhTObBmQTnKKByICmfWcwQnUcFT2FraWK7X6xJcsfWZm5SYj024VtMK6c4 kZb5ev6t7cvtQv2FFYvFZnW3Kn7RRQFAJ5zOyMc9JocZ9cZBNwiF7r51Qn1IGiwB eN5rN1iHr0qMlWz1zupWL3RTd57bWjauLrzyYRxbUZ05kUinzUlYrBb55f0IEAvD SwhNAHkb/HNzxfAhhvyIuQRqcpCiXkKcDQ/RyzxE0eltRlp5LyS7/YZ7gZXufwLa k0h74iXBKGrAvNzWeJrFOIOHyxBifp880cJqYS4mAEA7Q4hLiQ6lIX98FLCA9BVP cjdETQ== -----END CERTIFICATE----- pymongo-3.6.1/test/certificates/crl.pem0000644000076600000240000000431013245621354020345 0ustar shanestaff00000000000000Certificate Revocation List (CRL): Version 2 (0x1) Signature Algorithm: sha256WithRSAEncryption Issuer: /CN=Python Driver CA/OU=Drivers/O=MongoDB/L=Palo Alto/ST=California/C=US Last Update: Jan 31 23:00:50 2017 GMT Next Update: Jan 30 23:00:50 2022 GMT CRL extensions: X509v3 CRL Number: 4096 Revoked Certificates: Serial Number: 051740 Revocation Date: Jan 31 23:00:50 2017 GMT Serial Number: 087610 Revocation Date: Jan 31 23:00:50 2017 GMT Signature Algorithm: sha256WithRSAEncryption 50:c5:77:16:8c:6b:a7:07:9a:04:97:05:e0:48:47:cc:1c:92: a4:53:d6:12:91:5a:9e:b2:85:9a:20:57:cf:7c:09:58:ea:b2: 25:c0:8b:c5:69:64:73:d6:da:ce:41:a5:ad:c0:0a:b7:df:2a: 25:ca:59:fe:42:dd:d1:8c:56:d6:19:64:36:b7:2d:6e:f6:57: 6a:f9:a2:3d:4a:ea:f5:d1:09:56:6d:17:89:5e:0d:b5:02:3f: e6:f5:74:3e:ab:78:54:80:a6:f5:dd:ac:bc:a4:77:51:e9:a7: e9:98:98:0e:83:e2:96:36:0f:65:f1:1e:16:6d:fb:26:bd:e6: f9:13:dd:68:23:15:bb:59:d9:f9:c5:31:ec:b2:98:d0:a5:df: 45:a7:a2:4d:a5:ef:20:14:f3:bd:69:f1:e4:5b:39:f4:06:af: 98:17:15:97:5e:5f:0e:00:6f:1c:57:cc:48:e9:3f:e6:d2:de: 15:fa:dc:b9:d0:de:29:02:9a:7a:05:b9:c9:b0:8d:af:ec:b5: b4:76:fd:30:a4:6f:e4:d0:2b:dc:e5:dd:e3:ed:e9:b5:d9:d0: cc:e5:1b:85:0c:46:5f:d3:5e:94:c9:98:82:44:d7:e0:48:30: 9d:ec:a5:4c:e7:a3:bf:7f:c2:4f:36:74:ea:e2:a8:cb:57:6c: 44:b7:4f:ef -----BEGIN X509 CRL----- MIIB/TCB5gIBATANBgkqhkiG9w0BAQsFADB1MRkwFwYDVQQDExBQeXRob24gRHJp dmVyIENBMRAwDgYDVQQLEwdEcml2ZXJzMRAwDgYDVQQKEwdNb25nb0RCMRIwEAYD VQQHEwlQYWxvIEFsdG8xEzARBgNVBAgTCkNhbGlmb3JuaWExCzAJBgNVBAYTAlVT Fw0xNzAxMzEyMzAwNTBaFw0yMjAxMzAyMzAwNTBaMCwwFAIDBRdAFw0xNzAxMzEy MzAwNTBaMBQCAwh2EBcNMTcwMTMxMjMwMDUwWqAPMA0wCwYDVR0UBAQCAhAAMA0G CSqGSIb3DQEBCwUAA4IBAQBQxXcWjGunB5oElwXgSEfMHJKkU9YSkVqesoWaIFfP fAlY6rIlwIvFaWRz1trOQaWtwAq33yolyln+Qt3RjFbWGWQ2ty1u9ldq+aI9Sur1 0QlWbReJXg21Aj/m9XQ+q3hUgKb13ay8pHdR6afpmJgOg+KWNg9l8R4Wbfsmveb5 E91oIxW7Wdn5xTHsspjQpd9Fp6JNpe8gFPO9afHkWzn0Bq+YFxWXXl8OAG8cV8xI 6T/m0t4V+ty50N4pApp6BbnJsI2v7LW0dv0wpG/k0Cvc5d3j7em12dDM5RuFDEZf 016UyZiCRNfgSDCd7KVM56O/f8JPNnTq4qjLV2xEt0/v -----END X509 CRL----- pymongo-3.6.1/test/certificates/client.pem0000644000076600000240000000563513245615620021055 0ustar shanestaff00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAgDEaeJW3zMNQQUuhrjvpfZY+HSaPmuAW/oGPcyqZkDBHdZSz G6ZHtFmOs2uWC/5HWWm3cTIcEH5MtvMSR+SZb7Qym/8xsOfNVfLZeRjCMCtSq1OP OpFwl94wdnQU3tiEuzJvcPkYL3V8ZC7ccg/eOwS725BKheLNSHWGwRBC5cruvTT8 Hsdy45k713br3b6xde7xAaSwJEKgK+S+6eIBM1N1pW67i/L7gD2eIZ9IMdkWnQys 3/L6L4N6zxxWI45M21t0vvcvDVslQKCtg9IBT+04rO2+XCrOBZqQ7sIkLJgEeSQT yMtHkhFLmkqCh/518D1K/nS7scCsanACgjWAZQIDAQABAoIBABek9vT26IfkHpFs mc3BTzfS1nIaArLWUmdvDj53xQtLXGLboo2fBerF+gQt++cKZ66eSS+fxn1kTlIQ U3bfO03XNt5/G3M9zinkOSiU4HxyATWInAp5eQMKAHZYwuC3nwPIMbXsvAFE8kSo QBDpzFuma9/7pUM8qhCLXOePUEeB++I39rQc/H8P5BhRBWl1QjdhKBmgdiImp1w7 66fCQhi9yECpzszuuTgM87zCHhWRA692V/kZ9KmG5lPquU5V1pnmlpnZeWLbhRcT gYVDEl2zLqjH1muN05UvRvs6pU3hUZ2wJkYiW1bdS6t2DrHOt+LVGutgRC3YHrJD +9OYsAECgYEAtxDD9n4hOznnQoTRUQ4pEfnCwJCDXhnO7l3HVXN+gD3S2XV8RY5i xSJPe51rNb3G9xKEhdykRc9+bNTy8Vetm3OJIYANx0Tv76Q0zGqK4rNnRlbN9XPP zYiyGmFkVK35o96YbGYDmNXPg9ZnxtygkAGJHJ5A+4cF3FZURYu7O+UCgYEAs0On 5VnAdxx4z6Pctzi9h399Wr0wkt/+lyT1xN+lM3f2K2I48+Y8N5QvSG5kovqc2uoL THepOXgApxO49HIZ2pTHTU5/Rv4FubT5rLr7DlSR4ig99ZBVmT6rNnTvVareF273 UuEoOSENCSWbXB3zzYCkTYom4kLdyZcEOoPZ6oECgYEAot31mh0I76pKRvHnT5er jmSIkc2AMn3/ji48o5eHGgzmw7U0hKVy1jP6Rt38femDVFlTMczlEX7S2XxzT6hT uhOGExncKEb6uFd86ch4G8zEut2Liy4n2JIe8j1nh7bofJQTQ41W8eh3ILHlzhSH fNcA50ccRSsRWcAsZVs1jskCgYBp0Bmei2fL5wZ4qeRyUg+tl1dK5DlzuLvyQ2K+ nrRIuRqZaRTtAo2w0MtIGcFVYlCL93JL5KYCwg7AkCewQll4aDFsNFV0diSzBVwK 1SOzxb+GmvJXrKaaGh+9uTA8hOsrMdjTFwxDDHtBVu1pZlSujKxvCIkDLSz0/SsF zyYmAQKBgQCJSFwaNnehTBg5sXLG7ZeBjMOpFY9ZV1OHPl6QV60T/qdQYvKxDurf OAnb3wUVhjkZvzDIWpillp4WGd93nh0HWeeD5gAPJ2zDxE8lPWafovvqk+673Tni knxetqAv9fdRKbs6Dhligf14/X01/A5qDFHA9UbeQlsLL0tiksYjFg== -----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIDjTCCAnWgAwIBAgIDAlMVMA0GCSqGSIb3DQEBBQUAMHUxGTAXBgNVBAMTEFB5 dGhvbiBEcml2ZXIgQ0ExEDAOBgNVBAsTB0RyaXZlcnMxEDAOBgNVBAoTB01vbmdv REIxEjAQBgNVBAcTCVBhbG8gQWx0bzETMBEGA1UECBMKQ2FsaWZvcm5pYTELMAkG A1UEBhMCVVMwHhcNMTYwNTI0MDAyMDEzWhcNMzYwNTI0MDAyMDEzWjBkMQ8wDQYD VQQDEwZjbGllbnQxEDAOBgNVBAsTB0RyaXZlcnMxCTAHBgNVBAoTADESMBAGA1UE BxMJUGFsbyBBbHRvMRMwEQYDVQQIEwpDYWxpZm9ybmlhMQswCQYDVQQGEwJVUzCC ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIAxGniVt8zDUEFLoa476X2W Ph0mj5rgFv6Bj3MqmZAwR3WUsxumR7RZjrNrlgv+R1lpt3EyHBB+TLbzEkfkmW+0 Mpv/MbDnzVXy2XkYwjArUqtTjzqRcJfeMHZ0FN7YhLsyb3D5GC91fGQu3HIP3jsE u9uQSoXizUh1hsEQQuXK7r00/B7HcuOZO9d2692+sXXu8QGksCRCoCvkvuniATNT daVuu4vy+4A9niGfSDHZFp0MrN/y+i+Des8cViOOTNtbdL73Lw1bJUCgrYPSAU/t OKztvlwqzgWakO7CJCyYBHkkE8jLR5IRS5pKgof+dfA9Sv50u7HArGpwAoI1gGUC AwEAAaM3MDUwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMCMBEGA1Ud EQQKMAiCBmNsaWVudDANBgkqhkiG9w0BAQUFAAOCAQEAPOlVXQjEo2zd02PfbrpG 8+UTacyUbQheeCH7rCe118XuUTCipAqEhw9D6kQaudXva+yyg7SUd3NPMQv0d64J CJrH2MDYlI8nFwZoWbhPEJbAdimDaFa/p9OHfeEjNyolY9aPEni7forNF/OeP6Jx uFnd4f2tlJjIGqi+IBARlCmpnc5XG9TRvWLo0euMgG4bkBb5O46AqBaIzNyDNRUp LkGE5Nr6VLKw/38WXeU40n2vNrUeDMWB0w4c3+j7k1CpiEibbeYsjML/rDMOAH9q zR61mhExOxRkJJ1KRjrCbD4KFqIuND+XgwhGTymGr7vL6OKXxzdi2cbnwcNDRFiP jQ== -----END CERTIFICATE----- pymongo-3.6.1/test/certificates/ca.pem0000644000076600000240000000242613245615620020155 0ustar shanestaff00000000000000-----BEGIN CERTIFICATE----- MIIDkzCCAnugAwIBAgIDCHYQMA0GCSqGSIb3DQEBBQUAMHUxGTAXBgNVBAMTEFB5 dGhvbiBEcml2ZXIgQ0ExEDAOBgNVBAsTB0RyaXZlcnMxEDAOBgNVBAoTB01vbmdv REIxEjAQBgNVBAcTCVBhbG8gQWx0bzETMBEGA1UECBMKQ2FsaWZvcm5pYTELMAkG A1UEBhMCVVMwHhcNMTYwNTIzMjAxMzA4WhcNMzYwNTIzMjAxMzA4WjB1MRkwFwYD VQQDExBQeXRob24gRHJpdmVyIENBMRAwDgYDVQQLEwdEcml2ZXJzMRAwDgYDVQQK EwdNb25nb0RCMRIwEAYDVQQHEwlQYWxvIEFsdG8xEzARBgNVBAgTCkNhbGlmb3Ju aWExCzAJBgNVBAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA ijh4H/G+FDuqaawzLp9ry0kSIvZJCsRJmYwB8/JGXjxKBowCW5zavwUdWYrk0pH5 QRx9JvOwub0lpYTl26MLfNVoPe1zL+ahhyiDvobJqnQOPxxu86s56bNK+XZB7GBQ GoQrqm25yyODdRBRbDdeMqOT3pSGd7IkFdFWFVbs9Ro2GzUAc9SdfoAU3CJID9Up W3Gd/uxPIn0a/oYKPPbZzvAZ0QXVe4j01njOTn/i70btCayNOs/8uvwBEDSWGdGZ BHA+dfTaIG6cRnPCrhHUIKW0YIB3eOy3sdu1FUiFQXxrMLqVvaVxyfC6D1ubHAwu rK9ryguG2apWv4ODckoLzQIDAQABoywwKjAMBgNVHRMEBTADAQH/MBoGA1UdEQQT MBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAQEAXbuk/+gGYvLO isSHUdflWHJrG1d0TpvlxRdu8uDUPtG5qqp0b7L65jUI7SBu6YF3wa5Hs+xTPGnZ 4SG50VJ62CJckajVvrhNk1TS0NnZdCCE3PXWSm/CUJd+WQLmah5iOdb4WOGMiXoB ASzWee1yg9pxAxqZbqdlU/3Jk7sp98pxvz3FqQGsEWGegVUSmndGX+Dr3YbcEUrm DwAZdnjPV22N4HlbHXhZhn0YH9kN06mzsZND/fxw4tCkFbLH6juXyPuZnRj6TfN9 mbgNLP87kVnY1QOgMJl1uwTYQOK1SzMg973LHu1nKzw5VpY+F1gWZptP/3ZE+vpg I6CowZ3LEw== -----END CERTIFICATE----- pymongo-3.6.1/test/certificates/client_encrypted.pem0000644000076600000240000000612013245617773023133 0ustar shanestaff00000000000000-----BEGIN ENCRYPTED PRIVATE KEY----- MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIXVpiOphzP48CAggA MB0GCWCGSAFlAwQBAgQQCrPjZvdtfdmb5FnflM2QaQSCBNDwm1fRI4HsJ988MS6P 7xRkA8fscD/krgMsnG3g47NjgM+CtYRbUfKNMrgidajQVC74P7z1pQ+uHCz9Py7U 5Ge0uFFqkPyQl2RcnVjCdsx6ElCfdDLEnizVL1ERHMYTq0AXnYXyPN45Qpq9UfQG DPtoOAWpe6FSrQ0UFYZ3hTJIXK8A2J7phkPzrFz9CYtzaLv4ShD0osAOK0hN1hSp k258lb1oGifGc3BXv1atiXZI7R8tFhY5GlG/F6BksFHslOc01/Iix+rb04jV6aKB L1yyoUqjwO0Qt365FfUw/ZODwkW7n5gAT1hJ/qJc0Tkh92pv7ISnMpT5h40k4Q9L tkCm4aOymhSNWLjkBrgIVbLPgUo59bNZAgPrDz2Wjwt7rU3t5TyXPiScDYfxQyhj dhI7NkpC/nMGc/VRir8+zUSRvR6xIlkHM8glpVHWOU8gIRDv1RYQFhhecEFgoSMO oP54SK3a6A0BzrFZIRqMeVDXl1DXuA9tdgF7N2CIMdKTtAgOohTcTRESIk5ysFNm dY15ptDtj3woGsBffMgfa7AWxDq/97Gfp0leMdV7DTLHQWM/hjsmDaz7E5jRt36Z Qql2/SdUX3A91u4lvcCDgzcMe6YY1jJ3oPsuQRoMeZrj1jivR2TY0LUo474mxq3Y IxyP8wrGEhDDc3Tfq2TeFwyS2u1cxYu1JocRi1qxtbG/cdK340KXvdVJKl5mosys qUW0MdTyEAzzf77k14AqSOyXeVarttY4ZLMQd2qbW4kgBP7ZZ+Jz0YXPz4Pc0fiG ONeF8Q1LVVkneURHmvGXEoP+6rRk59neVtW1An5hxZkmeL9VPUm+q+cXAwLEJIxN tEzC6liIhzpMbyUFht6HbCaHvl6U4Zap56SqMfPXl6V6L3MDQC0uenQoPHmprYGP WcM1/F7BviQclHuyhdmkE/dbutJ8Dj7gKdv6qVYm4zS5BYcPfzqR/v4t4pXX+ubS Ng5GPZkrBCG4+D97/zSS4w6KPImbOsrQh6D0aJZKoRHcEgmEBaN0tDcwZQsSXQRs wB/ATl2sgebIRkNxABTqUyG3hnMMFNxoGPicv1P73Ai29056KrA5kq3Xz6PfOgI7 ZwfMgvwKNPewyzQ6cIoJHKCZellBeAm8B6SPI6fes4nLBvTJi6khjpK489nnZqYz MDIzu1eFPQLAVM2XB9ABTdZJJVM8f2OrjUMuXAiqj3eKV9A0lVUd7YfCjPmat3+3 JLlvJb7ePnVR7/O/4AgQatfIUyVRuqovR6s+j2KHu3iVEMUZvpEuQZpIzj+jptfC fSbaFoTjSGii97YKQ5OoZqnZiy3gkprY/SCkXa7Jlx7cb8h5pktbqEpAjlLd1MMd mES8HUiJpYSrseLCNJkbU5dKK1DqNj1UgAKRtU55HGYRGtpxx9zqPBqTiBkW7FwR qaBYhvM4STPj3cD0JqCE9TuJDG5/7jSKKKm342j1VyPN2VUXinreNDUFE2WePRdV OhyIa5zB4xuH5rLfqb3MJlxeuM7sMX/6Nor0+F7XfVn3HHEGGdjkclaJlFonmRTk +/xpROQ/iPCVWr85adnfkgBzIZOYDKFHqpXS0G4W0p09+eDfQWuHw1Nu5D3Gb9Y6 CrGuA90yQPSULpuE+GBjv8Od+Q== -----END ENCRYPTED PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIDjTCCAnWgAwIBAgIDAmB0MA0GCSqGSIb3DQEBBQUAMHUxGTAXBgNVBAMTEFB5 dGhvbiBEcml2ZXIgQ0ExEDAOBgNVBAsTB0RyaXZlcnMxEDAOBgNVBAoTB01vbmdv REIxEjAQBgNVBAcTCVBhbG8gQWx0bzETMBEGA1UECBMKQ2FsaWZvcm5pYTELMAkG A1UEBhMCVVMwHhcNMTYwNTI0MDAyMTQ4WhcNMzYwNTI0MDAyMTQ4WjBkMQ8wDQYD VQQDEwZjbGllbnQxEDAOBgNVBAsTB0RyaXZlcnMxCTAHBgNVBAoTADESMBAGA1UE BxMJUGFsbyBBbHRvMRMwEQYDVQQIEwpDYWxpZm9ybmlhMQswCQYDVQQGEwJVUzCC ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALuWTvDI4CRfGVGkvRnGfwEp j2U/fwMvZoJW3LT0w2nQRUsa/hrf74Ggm9FbhaaM9bxA6eS5vWEaaJX9gfEEqOaj FlF8O++7+8+CrxsBWRK6yNT8nkB9rg3VCv5gqXJr9rpRhWAeq3QuFhunACrk+Mt4 5KL+ueYerJyhJwx87LiwGAMxasdr+U8j/s5kc+SIhwQxWXYVOSgdweJd4FlEGYvZ 0WR/oheEWp1ayZzf4cE8K2aD0YC6CS2ErFke6W4ZsdqtwGRjLwz71SMG0tSEpOjg NkNJBqNju3FU9AtFJFS7t9wPzMG/2LIioSXKe+RW4/SBjhc0LM7xgDM9oFYqZ3sC AwEAAaM3MDUwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMCMBEGA1Ud EQQKMAiCBmNsaWVudDANBgkqhkiG9w0BAQUFAAOCAQEAMJgtYqm4cgOCmFiacwYy ZtpWJHLk1zFwBLrNdQu9GKlYhC+903yV/VypnRjOd0McdopMZ+4aQut4CE01PLty FEZ6UrpqXbEFniGh5PKaouhqrX896iBwPRn5eeFKf40sFgdNReJ3KDyGt6kqTWn6 rY13wgopMy0A3NfCqmBHHXYRBRl9GS0O2bOOLMR49o7iBW+Ga638/Z3+4xx8T9Mh +ejauV8EWZFKfg43soXuGHLDfDl3BMqH4iP12y+e8NtBEMRl6RPbZcO3X9Zkavba rmlcdR/01UK9/FiWP2rPNNxk2ypqZDxkPzT8sCDBQXtRgemxTlxgeKryDgXlF1iD Pw== -----END CERTIFICATE----- pymongo-3.6.1/test/test_cursor.py0000644000076600000240000016434713245621354017364 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the cursor module.""" import copy import gc import itertools import random import re import sys import time import threading sys.path[0:0] = [""] from bson import decode_all from bson.code import Code from bson.py3compat import PY3 from bson.son import SON from pymongo import (monitoring, ASCENDING, DESCENDING, ALL, OFF) from pymongo.collation import Collation from pymongo.cursor import CursorType from pymongo.errors import (ConfigurationError, ExecutionTimeout, InvalidOperation, OperationFailure) from pymongo.read_concern import ReadConcern from test import (client_context, SkipTest, unittest, IntegrationTest, Version) from test.utils import (EventListener, ignore_deprecations, rs_or_single_client, WhiteListEventListener) if PY3: long = int class TestCursor(IntegrationTest): def test_deepcopy_cursor_littered_with_regexes(self): cursor = self.db.test.find({ "x": re.compile("^hmmm.*"), "y": [re.compile("^hmm.*")], "z": {"a": [re.compile("^hm.*")]}, re.compile("^key.*"): {"a": [re.compile("^hm.*")]}}) cursor2 = copy.deepcopy(cursor) self.assertEqual(cursor._Cursor__spec, cursor2._Cursor__spec) def test_add_remove_option(self): cursor = self.db.test.find() self.assertEqual(0, cursor._Cursor__query_flags) cursor.add_option(2) cursor2 = self.db.test.find(cursor_type=CursorType.TAILABLE) self.assertEqual(2, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.add_option(32) cursor2 = self.db.test.find(cursor_type=CursorType.TAILABLE_AWAIT) self.assertEqual(34, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.add_option(128) cursor2 = self.db.test.find( cursor_type=CursorType.TAILABLE_AWAIT).add_option(128) self.assertEqual(162, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) self.assertEqual(162, cursor._Cursor__query_flags) cursor.add_option(128) self.assertEqual(162, cursor._Cursor__query_flags) cursor.remove_option(128) cursor2 = self.db.test.find(cursor_type=CursorType.TAILABLE_AWAIT) self.assertEqual(34, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.remove_option(32) cursor2 = self.db.test.find(cursor_type=CursorType.TAILABLE) self.assertEqual(2, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) self.assertEqual(2, cursor._Cursor__query_flags) cursor.remove_option(32) self.assertEqual(2, cursor._Cursor__query_flags) # Timeout cursor = self.db.test.find(no_cursor_timeout=True) self.assertEqual(16, cursor._Cursor__query_flags) cursor2 = self.db.test.find().add_option(16) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.remove_option(16) self.assertEqual(0, cursor._Cursor__query_flags) # Tailable / Await data cursor = self.db.test.find(cursor_type=CursorType.TAILABLE_AWAIT) self.assertEqual(34, cursor._Cursor__query_flags) cursor2 = self.db.test.find().add_option(34) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.remove_option(32) self.assertEqual(2, cursor._Cursor__query_flags) # Partial cursor = self.db.test.find(allow_partial_results=True) self.assertEqual(128, cursor._Cursor__query_flags) cursor2 = self.db.test.find().add_option(128) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.remove_option(128) self.assertEqual(0, cursor._Cursor__query_flags) def test_add_remove_option_exhaust(self): # Exhaust - which mongos doesn't support if client_context.is_mongos: with self.assertRaises(InvalidOperation): self.db.test.find(cursor_type=CursorType.EXHAUST) else: cursor = self.db.test.find(cursor_type=CursorType.EXHAUST) self.assertEqual(64, cursor._Cursor__query_flags) cursor2 = self.db.test.find().add_option(64) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) self.assertTrue(cursor._Cursor__exhaust) cursor.remove_option(64) self.assertEqual(0, cursor._Cursor__query_flags) self.assertFalse(cursor._Cursor__exhaust) def test_max_time_ms(self): db = self.db db.pymongo_test.drop() coll = db.pymongo_test self.assertRaises(TypeError, coll.find().max_time_ms, 'foo') coll.insert_one({"amalia": 1}) coll.insert_one({"amalia": 2}) coll.find().max_time_ms(None) coll.find().max_time_ms(long(1)) cursor = coll.find().max_time_ms(999) self.assertEqual(999, cursor._Cursor__max_time_ms) cursor = coll.find().max_time_ms(10).max_time_ms(1000) self.assertEqual(1000, cursor._Cursor__max_time_ms) cursor = coll.find().max_time_ms(999) c2 = cursor.clone() self.assertEqual(999, c2._Cursor__max_time_ms) self.assertTrue("$maxTimeMS" in cursor._Cursor__query_spec()) self.assertTrue("$maxTimeMS" in c2._Cursor__query_spec()) self.assertTrue(coll.find_one(max_time_ms=1000)) client = self.client if (not client_context.is_mongos and client_context.test_commands_enabled): # Cursor parses server timeout error in response to initial query. client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") try: cursor = coll.find().max_time_ms(1) try: next(cursor) except ExecutionTimeout: pass else: self.fail("ExecutionTimeout not raised") self.assertRaises(ExecutionTimeout, coll.find_one, max_time_ms=1) finally: client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") @client_context.require_version_min(3, 1, 9, -1) def test_max_await_time_ms(self): db = self.db db.pymongo_test.drop() coll = db.create_collection("pymongo_test", capped=True, size=4096) self.assertRaises(TypeError, coll.find().max_await_time_ms, 'foo') coll.insert_one({"amalia": 1}) coll.insert_one({"amalia": 2}) coll.find().max_await_time_ms(None) coll.find().max_await_time_ms(long(1)) # When cursor is not tailable_await cursor = coll.find() self.assertEqual(None, cursor._Cursor__max_await_time_ms) cursor = coll.find().max_await_time_ms(99) self.assertEqual(None, cursor._Cursor__max_await_time_ms) # If cursor is tailable_await and timeout is unset cursor = coll.find(cursor_type=CursorType.TAILABLE_AWAIT) self.assertEqual(None, cursor._Cursor__max_await_time_ms) # If cursor is tailable_await and timeout is set cursor = coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_await_time_ms(99) self.assertEqual(99, cursor._Cursor__max_await_time_ms) cursor = coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_await_time_ms( 10).max_await_time_ms(90) self.assertEqual(90, cursor._Cursor__max_await_time_ms) listener = WhiteListEventListener('find', 'getMore') saved_listeners = monitoring._LISTENERS monitoring._LISTENERS = monitoring._Listeners([], [], [], []) coll = rs_or_single_client( event_listeners=[listener])[self.db.name].pymongo_test results = listener.results try: # Tailable_await defaults. list(coll.find(cursor_type=CursorType.TAILABLE_AWAIT)) # find self.assertFalse('maxTimeMS' in results['started'][0].command) # getMore self.assertFalse('maxTimeMS' in results['started'][1].command) results.clear() # Tailable_await with max_await_time_ms set. list(coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_await_time_ms(99)) # find self.assertEqual('find', results['started'][0].command_name) self.assertFalse('maxTimeMS' in results['started'][0].command) # getMore self.assertEqual('getMore', results['started'][1].command_name) self.assertTrue('maxTimeMS' in results['started'][1].command) self.assertEqual(99, results['started'][1].command['maxTimeMS']) results.clear() # Tailable_await with max_time_ms list(coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_time_ms(99)) # find self.assertEqual('find', results['started'][0].command_name) self.assertTrue('maxTimeMS' in results['started'][0].command) self.assertEqual(99, results['started'][0].command['maxTimeMS']) # getMore self.assertEqual('getMore', results['started'][1].command_name) self.assertFalse('maxTimeMS' in results['started'][1].command) results.clear() # Tailable_await with both max_time_ms and max_await_time_ms list(coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_time_ms( 99).max_await_time_ms(99)) # find self.assertEqual('find', results['started'][0].command_name) self.assertTrue('maxTimeMS' in results['started'][0].command) self.assertEqual(99, results['started'][0].command['maxTimeMS']) # getMore self.assertEqual('getMore', results['started'][1].command_name) self.assertTrue('maxTimeMS' in results['started'][1].command) self.assertEqual(99, results['started'][1].command['maxTimeMS']) results.clear() # Non tailable_await with max_await_time_ms list(coll.find(batch_size=1).max_await_time_ms(99)) # find self.assertEqual('find', results['started'][0].command_name) self.assertFalse('maxTimeMS' in results['started'][0].command) # getMore self.assertEqual('getMore', results['started'][1].command_name) self.assertFalse('maxTimeMS' in results['started'][1].command) results.clear() # Non tailable_await with max_time_ms list(coll.find(batch_size=1).max_time_ms(99)) # find self.assertEqual('find', results['started'][0].command_name) self.assertTrue('maxTimeMS' in results['started'][0].command) self.assertEqual(99, results['started'][0].command['maxTimeMS']) # getMore self.assertEqual('getMore', results['started'][1].command_name) self.assertFalse('maxTimeMS' in results['started'][1].command) # Non tailable_await with both max_time_ms and max_await_time_ms list(coll.find(batch_size=1).max_time_ms(99).max_await_time_ms(88)) # find self.assertEqual('find', results['started'][0].command_name) self.assertTrue('maxTimeMS' in results['started'][0].command) self.assertEqual(99, results['started'][0].command['maxTimeMS']) # getMore self.assertEqual('getMore', results['started'][1].command_name) self.assertFalse('maxTimeMS' in results['started'][1].command) finally: monitoring._LISTENERS = saved_listeners @client_context.require_test_commands @client_context.require_no_mongos def test_max_time_ms_getmore(self): # Test that Cursor handles server timeout error in response to getmore. coll = self.db.pymongo_test coll.insert_many([{} for _ in range(200)]) cursor = coll.find().max_time_ms(100) # Send initial query before turning on failpoint. next(cursor) self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") try: try: # Iterate up to first getmore. list(cursor) except ExecutionTimeout: pass else: self.fail("ExecutionTimeout not raised") finally: self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") def test_explain(self): a = self.db.test.find() a.explain() for _ in a: break b = a.explain() # "cursor" pre MongoDB 2.7.6, "executionStats" post self.assertTrue("cursor" in b or "executionStats" in b) def test_explain_with_read_concern(self): # Do not add readConcern level to explain. listener = WhiteListEventListener("explain") client = rs_or_single_client(event_listeners=[listener]) self.addCleanup(client.close) coll = client.pymongo_test.test.with_options( read_concern=ReadConcern(level="local")) self.assertTrue(coll.find().explain()) started = listener.results['started'] self.assertEqual(len(started), 1) self.assertNotIn("readConern", started[0].command) def test_hint(self): db = self.db self.assertRaises(TypeError, db.test.find().hint, 5.5) db.test.drop() db.test.insert_many([{"num": i, "foo": i} for i in range(100)]) self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}) .hint([("num", ASCENDING)]).explain) self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}) .hint([("foo", ASCENDING)]).explain) spec = [("num", DESCENDING)] index = db.test.create_index(spec) first = next(db.test.find()) self.assertEqual(0, first.get('num')) first = next(db.test.find().hint(spec)) self.assertEqual(99, first.get('num')) self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}) .hint([("foo", ASCENDING)]).explain) a = db.test.find({"num": 17}) a.hint(spec) for _ in a: break self.assertRaises(InvalidOperation, a.hint, spec) def test_hint_by_name(self): db = self.db db.test.drop() db.test.insert_many([{"i": i} for i in range(100)]) db.test.create_index([('i', DESCENDING)], name='fooindex') first = next(db.test.find()) self.assertEqual(0, first.get('i')) first = next(db.test.find().hint('fooindex')) self.assertEqual(99, first.get('i')) def test_limit(self): db = self.db self.assertRaises(TypeError, db.test.find().limit, None) self.assertRaises(TypeError, db.test.find().limit, "hello") self.assertRaises(TypeError, db.test.find().limit, 5.5) self.assertTrue(db.test.find().limit(long(5))) db.test.drop() db.test.insert_many([{"x": i} for i in range(100)]) count = 0 for _ in db.test.find(): count += 1 self.assertEqual(count, 100) count = 0 for _ in db.test.find().limit(20): count += 1 self.assertEqual(count, 20) count = 0 for _ in db.test.find().limit(99): count += 1 self.assertEqual(count, 99) count = 0 for _ in db.test.find().limit(1): count += 1 self.assertEqual(count, 1) count = 0 for _ in db.test.find().limit(0): count += 1 self.assertEqual(count, 100) count = 0 for _ in db.test.find().limit(0).limit(50).limit(10): count += 1 self.assertEqual(count, 10) a = db.test.find() a.limit(10) for _ in a: break self.assertRaises(InvalidOperation, a.limit, 5) def test_max(self): db = self.db db.test.drop() db.test.create_index([("j", ASCENDING)]) db.test.insert_many([{"j": j, "k": j} for j in range(10)]) cursor = db.test.find().max([("j", 3)]) self.assertEqual(len(list(cursor)), 3) # Tuple. cursor = db.test.find().max((("j", 3), )) self.assertEqual(len(list(cursor)), 3) # Compound index. db.test.create_index([("j", ASCENDING), ("k", ASCENDING)]) cursor = db.test.find().max([("j", 3), ("k", 3)]) self.assertEqual(len(list(cursor)), 3) # Wrong order. cursor = db.test.find().max([("k", 3), ("j", 3)]) self.assertRaises(OperationFailure, list, cursor) # No such index. cursor = db.test.find().max([("k", 3)]) self.assertRaises(OperationFailure, list, cursor) self.assertRaises(TypeError, db.test.find().max, 10) self.assertRaises(TypeError, db.test.find().max, {"j": 10}) def test_min(self): db = self.db db.test.drop() db.test.create_index([("j", ASCENDING)]) db.test.insert_many([{"j": j, "k": j} for j in range(10)]) cursor = db.test.find().min([("j", 3)]) self.assertEqual(len(list(cursor)), 7) # Tuple. cursor = db.test.find().min((("j", 3), )) self.assertEqual(len(list(cursor)), 7) # Compound index. db.test.create_index([("j", ASCENDING), ("k", ASCENDING)]) cursor = db.test.find().min([("j", 3), ("k", 3)]) self.assertEqual(len(list(cursor)), 7) # Wrong order. cursor = db.test.find().min([("k", 3), ("j", 3)]) self.assertRaises(OperationFailure, list, cursor) # No such index. cursor = db.test.find().min([("k", 3)]) self.assertRaises(OperationFailure, list, cursor) self.assertRaises(TypeError, db.test.find().min, 10) self.assertRaises(TypeError, db.test.find().min, {"j": 10}) def test_batch_size(self): db = self.db db.test.drop() db.test.insert_many([{"x": x} for x in range(200)]) self.assertRaises(TypeError, db.test.find().batch_size, None) self.assertRaises(TypeError, db.test.find().batch_size, "hello") self.assertRaises(TypeError, db.test.find().batch_size, 5.5) self.assertRaises(ValueError, db.test.find().batch_size, -1) self.assertTrue(db.test.find().batch_size(long(5))) a = db.test.find() for _ in a: break self.assertRaises(InvalidOperation, a.batch_size, 5) def cursor_count(cursor, expected_count): count = 0 for _ in cursor: count += 1 self.assertEqual(expected_count, count) cursor_count(db.test.find().batch_size(0), 200) cursor_count(db.test.find().batch_size(1), 200) cursor_count(db.test.find().batch_size(2), 200) cursor_count(db.test.find().batch_size(5), 200) cursor_count(db.test.find().batch_size(100), 200) cursor_count(db.test.find().batch_size(500), 200) cursor_count(db.test.find().batch_size(0).limit(1), 1) cursor_count(db.test.find().batch_size(1).limit(1), 1) cursor_count(db.test.find().batch_size(2).limit(1), 1) cursor_count(db.test.find().batch_size(5).limit(1), 1) cursor_count(db.test.find().batch_size(100).limit(1), 1) cursor_count(db.test.find().batch_size(500).limit(1), 1) cursor_count(db.test.find().batch_size(0).limit(10), 10) cursor_count(db.test.find().batch_size(1).limit(10), 10) cursor_count(db.test.find().batch_size(2).limit(10), 10) cursor_count(db.test.find().batch_size(5).limit(10), 10) cursor_count(db.test.find().batch_size(100).limit(10), 10) cursor_count(db.test.find().batch_size(500).limit(10), 10) cur = db.test.find().batch_size(1) next(cur) if client_context.version.at_least(3, 1, 9): # find command batchSize should be 1 self.assertEqual(0, len(cur._Cursor__data)) else: # OP_QUERY ntoreturn should be 2 self.assertEqual(1, len(cur._Cursor__data)) next(cur) self.assertEqual(0, len(cur._Cursor__data)) next(cur) self.assertEqual(0, len(cur._Cursor__data)) next(cur) self.assertEqual(0, len(cur._Cursor__data)) def test_limit_and_batch_size(self): db = self.db db.test.drop() db.test.insert_many([{"x": x} for x in range(500)]) curs = db.test.find().limit(0).batch_size(10) next(curs) self.assertEqual(10, curs._Cursor__retrieved) curs = db.test.find(limit=0, batch_size=10) next(curs) self.assertEqual(10, curs._Cursor__retrieved) curs = db.test.find().limit(-2).batch_size(0) next(curs) self.assertEqual(2, curs._Cursor__retrieved) curs = db.test.find(limit=-2, batch_size=0) next(curs) self.assertEqual(2, curs._Cursor__retrieved) curs = db.test.find().limit(-4).batch_size(5) next(curs) self.assertEqual(4, curs._Cursor__retrieved) curs = db.test.find(limit=-4, batch_size=5) next(curs) self.assertEqual(4, curs._Cursor__retrieved) curs = db.test.find().limit(50).batch_size(500) next(curs) self.assertEqual(50, curs._Cursor__retrieved) curs = db.test.find(limit=50, batch_size=500) next(curs) self.assertEqual(50, curs._Cursor__retrieved) curs = db.test.find().batch_size(500) next(curs) self.assertEqual(500, curs._Cursor__retrieved) curs = db.test.find(batch_size=500) next(curs) self.assertEqual(500, curs._Cursor__retrieved) curs = db.test.find().limit(50) next(curs) self.assertEqual(50, curs._Cursor__retrieved) curs = db.test.find(limit=50) next(curs) self.assertEqual(50, curs._Cursor__retrieved) # these two might be shaky, as the default # is set by the server. as of 2.0.0-rc0, 101 # or 1MB (whichever is smaller) is default # for queries without ntoreturn curs = db.test.find() next(curs) self.assertEqual(101, curs._Cursor__retrieved) curs = db.test.find().limit(0).batch_size(0) next(curs) self.assertEqual(101, curs._Cursor__retrieved) curs = db.test.find(limit=0, batch_size=0) next(curs) self.assertEqual(101, curs._Cursor__retrieved) def test_skip(self): db = self.db self.assertRaises(TypeError, db.test.find().skip, None) self.assertRaises(TypeError, db.test.find().skip, "hello") self.assertRaises(TypeError, db.test.find().skip, 5.5) self.assertRaises(ValueError, db.test.find().skip, -5) self.assertTrue(db.test.find().skip(long(5))) db.drop_collection("test") db.test.insert_many([{"x": i} for i in range(100)]) for i in db.test.find(): self.assertEqual(i["x"], 0) break for i in db.test.find().skip(20): self.assertEqual(i["x"], 20) break for i in db.test.find().skip(99): self.assertEqual(i["x"], 99) break for i in db.test.find().skip(1): self.assertEqual(i["x"], 1) break for i in db.test.find().skip(0): self.assertEqual(i["x"], 0) break for i in db.test.find().skip(0).skip(50).skip(10): self.assertEqual(i["x"], 10) break for i in db.test.find().skip(1000): self.fail() a = db.test.find() a.skip(10) for _ in a: break self.assertRaises(InvalidOperation, a.skip, 5) def test_sort(self): db = self.db self.assertRaises(TypeError, db.test.find().sort, 5) self.assertRaises(ValueError, db.test.find().sort, []) self.assertRaises(TypeError, db.test.find().sort, [], ASCENDING) self.assertRaises(TypeError, db.test.find().sort, [("hello", DESCENDING)], DESCENDING) db.test.drop() unsort = list(range(10)) random.shuffle(unsort) db.test.insert_many([{"x": i} for i in unsort]) asc = [i["x"] for i in db.test.find().sort("x", ASCENDING)] self.assertEqual(asc, list(range(10))) asc = [i["x"] for i in db.test.find().sort("x")] self.assertEqual(asc, list(range(10))) asc = [i["x"] for i in db.test.find().sort([("x", ASCENDING)])] self.assertEqual(asc, list(range(10))) expect = list(reversed(range(10))) desc = [i["x"] for i in db.test.find().sort("x", DESCENDING)] self.assertEqual(desc, expect) desc = [i["x"] for i in db.test.find().sort([("x", DESCENDING)])] self.assertEqual(desc, expect) desc = [i["x"] for i in db.test.find().sort("x", ASCENDING).sort("x", DESCENDING)] self.assertEqual(desc, expect) expected = [(1, 5), (2, 5), (0, 3), (7, 3), (9, 2), (2, 1), (3, 1)] shuffled = list(expected) random.shuffle(shuffled) db.test.drop() for (a, b) in shuffled: db.test.insert_one({"a": a, "b": b}) result = [(i["a"], i["b"]) for i in db.test.find().sort([("b", DESCENDING), ("a", ASCENDING)])] self.assertEqual(result, expected) a = db.test.find() a.sort("x", ASCENDING) for _ in a: break self.assertRaises(InvalidOperation, a.sort, "x", ASCENDING) def test_count(self): db = self.db db.test.drop() self.assertEqual(0, db.test.find().count()) db.test.insert_many([{"x": i} for i in range(10)]) self.assertEqual(10, db.test.find().count()) self.assertTrue(isinstance(db.test.find().count(), int)) self.assertEqual(10, db.test.find().limit(5).count()) self.assertEqual(10, db.test.find().skip(5).count()) self.assertEqual(1, db.test.find({"x": 1}).count()) self.assertEqual(5, db.test.find({"x": {"$lt": 5}}).count()) a = db.test.find() b = a.count() for _ in a: break self.assertEqual(b, a.count()) self.assertEqual(0, db.test.acollectionthatdoesntexist.find().count()) def test_count_with_hint(self): collection = self.db.test collection.drop() collection.insert_many([{'i': 1}, {'i': 2}]) self.assertEqual(2, collection.find().count()) collection.create_index([('i', 1)]) self.assertEqual(1, collection.find({'i': 1}).hint("_id_").count()) self.assertEqual(2, collection.find().hint("_id_").count()) self.assertRaises(OperationFailure, collection.find({'i': 1}).hint("BAD HINT").count) # Create a sparse index which should have no entries. collection.create_index([('x', 1)], sparse=True) self.assertEqual(0, collection.find({'i': 1}).hint("x_1").count()) self.assertEqual( 0, collection.find({'i': 1}).hint([("x", 1)]).count()) if client_context.version.at_least(3, 3, 2): self.assertEqual(0, collection.find().hint("x_1").count()) self.assertEqual(0, collection.find().hint([("x", 1)]).count()) else: self.assertEqual(2, collection.find().hint("x_1").count()) self.assertEqual(2, collection.find().hint([("x", 1)]).count()) def test_where(self): db = self.db db.test.drop() a = db.test.find() self.assertRaises(TypeError, a.where, 5) self.assertRaises(TypeError, a.where, None) self.assertRaises(TypeError, a.where, {}) db.test.insert_many([{"x": i} for i in range(10)]) self.assertEqual(3, len(list(db.test.find().where('this.x < 3')))) self.assertEqual(3, len(list(db.test.find().where(Code('this.x < 3'))))) self.assertEqual(3, len(list(db.test.find().where(Code('this.x < i', {"i": 3}))))) self.assertEqual(10, len(list(db.test.find()))) self.assertEqual(3, db.test.find().where('this.x < 3').count()) self.assertEqual(10, db.test.find().count()) self.assertEqual(3, db.test.find().where(u'this.x < 3').count()) self.assertEqual([0, 1, 2], [a["x"] for a in db.test.find().where('this.x < 3')]) self.assertEqual([], [a["x"] for a in db.test.find({"x": 5}).where('this.x < 3')]) self.assertEqual([5], [a["x"] for a in db.test.find({"x": 5}).where('this.x > 3')]) cursor = db.test.find().where('this.x < 3').where('this.x > 7') self.assertEqual([8, 9], [a["x"] for a in cursor]) a = db.test.find() b = a.where('this.x > 3') for _ in a: break self.assertRaises(InvalidOperation, a.where, 'this.x < 3') def test_rewind(self): self.db.test.insert_many([{"x": i} for i in range(1, 4)]) cursor = self.db.test.find().limit(2) count = 0 for _ in cursor: count += 1 self.assertEqual(2, count) count = 0 for _ in cursor: count += 1 self.assertEqual(0, count) cursor.rewind() count = 0 for _ in cursor: count += 1 self.assertEqual(2, count) cursor.rewind() count = 0 for _ in cursor: break cursor.rewind() for _ in cursor: count += 1 self.assertEqual(2, count) self.assertEqual(cursor, cursor.rewind()) def test_clone(self): self.db.test.insert_many([{"x": i} for i in range(1, 4)]) cursor = self.db.test.find().limit(2) count = 0 for _ in cursor: count += 1 self.assertEqual(2, count) count = 0 for _ in cursor: count += 1 self.assertEqual(0, count) cursor = cursor.clone() cursor2 = cursor.clone() count = 0 for _ in cursor: count += 1 self.assertEqual(2, count) for _ in cursor2: count += 1 self.assertEqual(4, count) cursor.rewind() count = 0 for _ in cursor: break cursor = cursor.clone() for _ in cursor: count += 1 self.assertEqual(2, count) self.assertNotEqual(cursor, cursor.clone()) # Just test attributes cursor = self.db.test.find({"x": re.compile("^hello.*")}, skip=1, no_cursor_timeout=True, cursor_type=CursorType.TAILABLE_AWAIT, allow_partial_results=True, manipulate=False, projection={'_id': False}).limit(2) cursor.min([('a', 1)]).max([('b', 3)]) cursor.add_option(128) cursor.comment('hi!') cursor2 = cursor.clone() self.assertEqual(cursor._Cursor__skip, cursor2._Cursor__skip) self.assertEqual(cursor._Cursor__limit, cursor2._Cursor__limit) self.assertEqual(type(cursor._Cursor__codec_options), type(cursor2._Cursor__codec_options)) self.assertEqual(cursor._Cursor__manipulate, cursor2._Cursor__manipulate) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__comment, cursor2._Cursor__comment) self.assertEqual(cursor._Cursor__min, cursor2._Cursor__min) self.assertEqual(cursor._Cursor__max, cursor2._Cursor__max) # Shallow copies can so can mutate cursor2 = copy.copy(cursor) cursor2._Cursor__projection['cursor2'] = False self.assertTrue('cursor2' in cursor._Cursor__projection) # Deepcopies and shouldn't mutate cursor3 = copy.deepcopy(cursor) cursor3._Cursor__projection['cursor3'] = False self.assertFalse('cursor3' in cursor._Cursor__projection) cursor4 = cursor.clone() cursor4._Cursor__projection['cursor4'] = False self.assertFalse('cursor4' in cursor._Cursor__projection) # Test memo when deepcopying queries query = {"hello": "world"} query["reflexive"] = query cursor = self.db.test.find(query) cursor2 = copy.deepcopy(cursor) self.assertNotEqual(id(cursor._Cursor__spec), id(cursor2._Cursor__spec)) self.assertEqual(id(cursor2._Cursor__spec['reflexive']), id(cursor2._Cursor__spec)) self.assertEqual(len(cursor2._Cursor__spec), 2) # Ensure hints are cloned as the correct type cursor = self.db.test.find().hint([('z', 1), ("a", 1)]) cursor2 = copy.deepcopy(cursor) self.assertTrue(isinstance(cursor2._Cursor__hint, SON)) self.assertEqual(cursor._Cursor__hint, cursor2._Cursor__hint) def test_count_with_fields(self): self.db.test.drop() self.db.test.insert_one({"x": 1}) self.assertEqual(1, self.db.test.find({}, ["a"]).count()) def test_bad_getitem(self): self.assertRaises(TypeError, lambda x: self.db.test.find()[x], "hello") self.assertRaises(TypeError, lambda x: self.db.test.find()[x], 5.5) self.assertRaises(TypeError, lambda x: self.db.test.find()[x], None) def test_getitem_slice_index(self): self.db.drop_collection("test") self.db.test.insert_many([{"i": i} for i in range(100)]) count = itertools.count self.assertRaises(IndexError, lambda: self.db.test.find()[-1:]) self.assertRaises(IndexError, lambda: self.db.test.find()[1:2:2]) for a, b in zip(count(0), self.db.test.find()): self.assertEqual(a, b['i']) self.assertEqual(100, len(list(self.db.test.find()[0:]))) for a, b in zip(count(0), self.db.test.find()[0:]): self.assertEqual(a, b['i']) self.assertEqual(80, len(list(self.db.test.find()[20:]))) for a, b in zip(count(20), self.db.test.find()[20:]): self.assertEqual(a, b['i']) for a, b in zip(count(99), self.db.test.find()[99:]): self.assertEqual(a, b['i']) for i in self.db.test.find()[1000:]: self.fail() self.assertEqual(5, len(list(self.db.test.find()[20:25]))) self.assertEqual(5, len(list( self.db.test.find()[long(20):long(25)]))) for a, b in zip(count(20), self.db.test.find()[20:25]): self.assertEqual(a, b['i']) self.assertEqual(80, len(list(self.db.test.find()[40:45][20:]))) for a, b in zip(count(20), self.db.test.find()[40:45][20:]): self.assertEqual(a, b['i']) self.assertEqual(80, len(list(self.db.test.find()[40:45].limit(0).skip(20)) ) ) for a, b in zip(count(20), self.db.test.find()[40:45].limit(0).skip(20)): self.assertEqual(a, b['i']) self.assertEqual(80, len(list(self.db.test.find().limit(10).skip(40)[20:])) ) for a, b in zip(count(20), self.db.test.find().limit(10).skip(40)[20:]): self.assertEqual(a, b['i']) self.assertEqual(1, len(list(self.db.test.find()[:1]))) self.assertEqual(5, len(list(self.db.test.find()[:5]))) self.assertEqual(1, len(list(self.db.test.find()[99:100]))) self.assertEqual(1, len(list(self.db.test.find()[99:1000]))) self.assertEqual(0, len(list(self.db.test.find()[10:10]))) self.assertEqual(0, len(list(self.db.test.find()[:0]))) self.assertEqual(80, len(list(self.db.test.find()[10:10].limit(0).skip(20)) ) ) self.assertRaises(IndexError, lambda: self.db.test.find()[10:8]) def test_getitem_numeric_index(self): self.db.drop_collection("test") self.db.test.insert_many([{"i": i} for i in range(100)]) self.assertEqual(0, self.db.test.find()[0]['i']) self.assertEqual(50, self.db.test.find()[50]['i']) self.assertEqual(50, self.db.test.find().skip(50)[0]['i']) self.assertEqual(50, self.db.test.find().skip(49)[1]['i']) self.assertEqual(50, self.db.test.find()[long(50)]['i']) self.assertEqual(99, self.db.test.find()[99]['i']) self.assertRaises(IndexError, lambda x: self.db.test.find()[x], -1) self.assertRaises(IndexError, lambda x: self.db.test.find()[x], 100) self.assertRaises(IndexError, lambda x: self.db.test.find().skip(50)[x], 50) def test_count_with_limit_and_skip(self): self.assertRaises(TypeError, self.db.test.find().count, "foo") def check_len(cursor, length): self.assertEqual(len(list(cursor)), cursor.count(True)) self.assertEqual(length, cursor.count(True)) self.db.drop_collection("test") self.db.test.insert_many([{"i": i} for i in range(100)]) check_len(self.db.test.find(), 100) check_len(self.db.test.find().limit(10), 10) check_len(self.db.test.find().limit(110), 100) check_len(self.db.test.find().skip(10), 90) check_len(self.db.test.find().skip(110), 0) check_len(self.db.test.find().limit(10).skip(10), 10) check_len(self.db.test.find()[10:20], 10) check_len(self.db.test.find().limit(10).skip(95), 5) check_len(self.db.test.find()[95:105], 5) def test_len(self): self.assertRaises(TypeError, len, self.db.test.find()) def test_properties(self): self.assertEqual(self.db.test, self.db.test.find().collection) def set_coll(): self.db.test.find().collection = "hello" self.assertRaises(AttributeError, set_coll) def test_get_more(self): db = self.db db.drop_collection("test") db.test.insert_many([{'i': i} for i in range(10)]) self.assertEqual(10, len(list(db.test.find().batch_size(5)))) def test_tailable(self): db = self.db db.drop_collection("test") db.create_collection("test", capped=True, size=1000, max=3) self.addCleanup(db.drop_collection, "test") cursor = db.test.find(cursor_type=CursorType.TAILABLE) db.test.insert_one({"x": 1}) count = 0 for doc in cursor: count += 1 self.assertEqual(1, doc["x"]) self.assertEqual(1, count) db.test.insert_one({"x": 2}) count = 0 for doc in cursor: count += 1 self.assertEqual(2, doc["x"]) self.assertEqual(1, count) db.test.insert_one({"x": 3}) count = 0 for doc in cursor: count += 1 self.assertEqual(3, doc["x"]) self.assertEqual(1, count) # Capped rollover - the collection can never # have more than 3 documents. Just make sure # this doesn't raise... db.test.insert_many([{"x": i} for i in range(4, 7)]) self.assertEqual(0, len(list(cursor))) # and that the cursor doesn't think it's still alive. self.assertFalse(cursor.alive) self.assertEqual(3, db.test.count()) # __getitem__(index) for cursor in (db.test.find(cursor_type=CursorType.TAILABLE), db.test.find(cursor_type=CursorType.TAILABLE_AWAIT)): self.assertEqual(4, cursor[0]["x"]) self.assertEqual(5, cursor[1]["x"]) self.assertEqual(6, cursor[2]["x"]) cursor.rewind() self.assertEqual([4], [doc["x"] for doc in cursor[0:1]]) cursor.rewind() self.assertEqual([5], [doc["x"] for doc in cursor[1:2]]) cursor.rewind() self.assertEqual([6], [doc["x"] for doc in cursor[2:3]]) cursor.rewind() self.assertEqual([4, 5], [doc["x"] for doc in cursor[0:2]]) cursor.rewind() self.assertEqual([5, 6], [doc["x"] for doc in cursor[1:3]]) cursor.rewind() self.assertEqual([4, 5, 6], [doc["x"] for doc in cursor[0:3]]) def test_concurrent_close(self): """Ensure a tailable can be closed from another thread.""" db = self.db db.drop_collection("test") db.create_collection("test", capped=True, size=1000, max=3) self.addCleanup(db.drop_collection, "test") cursor = db.test.find(cursor_type=CursorType.TAILABLE) def iterate_cursor(): while cursor.alive: for doc in cursor: pass t = threading.Thread(target=iterate_cursor) t.start() time.sleep(1) cursor.close() self.assertFalse(cursor.alive) t.join(3) self.assertFalse(t.is_alive()) def test_distinct(self): self.db.drop_collection("test") self.db.test.insert_many( [{"a": 1}, {"a": 2}, {"a": 2}, {"a": 2}, {"a": 3}]) distinct = self.db.test.find({"a": {"$lt": 3}}).distinct("a") distinct.sort() self.assertEqual([1, 2], distinct) self.db.drop_collection("test") self.db.test.insert_one({"a": {"b": "a"}, "c": 12}) self.db.test.insert_one({"a": {"b": "b"}, "c": 8}) self.db.test.insert_one({"a": {"b": "c"}, "c": 12}) self.db.test.insert_one({"a": {"b": "c"}, "c": 8}) distinct = self.db.test.find({"c": 8}).distinct("a.b") distinct.sort() self.assertEqual(["b", "c"], distinct) def test_max_scan(self): self.db.drop_collection("test") self.db.test.insert_many([{} for _ in range(100)]) self.assertEqual(100, len(list(self.db.test.find()))) self.assertEqual(50, len(list(self.db.test.find().max_scan(50)))) self.assertEqual(50, len(list(self.db.test.find() .max_scan(90).max_scan(50)))) def test_with_statement(self): self.db.drop_collection("test") self.db.test.insert_many([{} for _ in range(100)]) c1 = self.db.test.find() with self.db.test.find() as c2: self.assertTrue(c2.alive) self.assertFalse(c2.alive) with self.db.test.find() as c2: self.assertEqual(100, len(list(c2))) self.assertFalse(c2.alive) self.assertTrue(c1.alive) @client_context.require_no_mongos def test_comment(self): if client_context.auth_enabled: raise SkipTest("SERVER-4754 - This test uses profiling.") # MongoDB 3.1.5 changed the ns for commands. regex = {'$regex': 'pymongo_test.(\$cmd|test)'} if client_context.version.at_least(3, 5, 8, -1): query_key = "command.comment" elif client_context.version.at_least(3, 1, 8, -1): query_key = "query.comment" else: query_key = "query.$comment" self.client.drop_database(self.db) self.db.set_profiling_level(ALL) try: list(self.db.test.find().comment('foo')) op = self.db.system.profile.find({'ns': 'pymongo_test.test', 'op': 'query', query_key: 'foo'}) self.assertEqual(op.count(), 1) self.db.test.find().comment('foo').count() op = self.db.system.profile.find({'ns': regex, 'op': 'command', 'command.count': 'test', 'command.$comment': 'foo'}) self.assertEqual(op.count(), 1) self.db.test.find().comment('foo').distinct('type') op = self.db.system.profile.find({'ns': regex, 'op': 'command', 'command.distinct': 'test', 'command.$comment': 'foo'}) self.assertEqual(op.count(), 1) finally: self.db.set_profiling_level(OFF) self.db.system.profile.drop() self.db.test.insert_many([{}, {}]) cursor = self.db.test.find() next(cursor) self.assertRaises(InvalidOperation, cursor.comment, 'hello') def test_modifiers(self): c = self.db.test # "modifiers" is deprecated. with ignore_deprecations(): cur = c.find() self.assertTrue('$query' not in cur._Cursor__query_spec()) cur = c.find().comment("testing").max_time_ms(500) self.assertTrue('$query' in cur._Cursor__query_spec()) self.assertEqual(cur._Cursor__query_spec()["$comment"], "testing") self.assertEqual(cur._Cursor__query_spec()["$maxTimeMS"], 500) cur = c.find( modifiers={"$maxTimeMS": 500, "$comment": "testing"}) self.assertTrue('$query' in cur._Cursor__query_spec()) self.assertEqual(cur._Cursor__query_spec()["$comment"], "testing") self.assertEqual(cur._Cursor__query_spec()["$maxTimeMS"], 500) # Keyword arg overwrites modifier. # If we remove the "modifiers" arg, delete this test after checking # that TestCommandMonitoring.test_find_options covers all cases. cur = c.find(comment="hi", modifiers={"$comment": "bye"}) self.assertEqual(cur._Cursor__query_spec()["$comment"], "hi") cur = c.find(max_scan=1, modifiers={"$maxScan": 2}) self.assertEqual(cur._Cursor__query_spec()["$maxScan"], 1) cur = c.find(max_time_ms=1, modifiers={"$maxTimeMS": 2}) self.assertEqual(cur._Cursor__query_spec()["$maxTimeMS"], 1) cur = c.find(min=1, modifiers={"$min": 2}) self.assertEqual(cur._Cursor__query_spec()["$min"], 1) cur = c.find(max=1, modifiers={"$max": 2}) self.assertEqual(cur._Cursor__query_spec()["$max"], 1) cur = c.find(return_key=True, modifiers={"$returnKey": False}) self.assertEqual(cur._Cursor__query_spec()["$returnKey"], True) cur = c.find(hint=[("a", 1)], modifiers={"$hint": {"b": "1"}}) self.assertEqual(cur._Cursor__query_spec()["$hint"], {"a": 1}) cur = c.find(snapshot=True, modifiers={"$snapshot": False}) self.assertEqual(cur._Cursor__query_spec()["$snapshot"], True) # The arg is named show_record_id after the "find" command arg, the # modifier is named $showDiskLoc for the OP_QUERY modifier. It's # stored as $showDiskLoc then upgraded to showRecordId if we send a # "find" command. cur = c.find(show_record_id=True, modifiers={"$showDiskLoc": False}) self.assertEqual(cur._Cursor__query_spec()["$showDiskLoc"], True) def test_alive(self): self.db.test.delete_many({}) self.db.test.insert_many([{} for _ in range(3)]) self.addCleanup(self.db.test.delete_many, {}) cursor = self.db.test.find().batch_size(2) n = 0 while True: cursor.next() n += 1 if 3 == n: self.assertFalse(cursor.alive) break self.assertTrue(cursor.alive) def test_close_kills_cursor_synchronously(self): # Kill any cursors possibly queued up by previous tests. gc.collect() self.client._process_periodic_tasks() listener = WhiteListEventListener("killCursors") results = listener.results client = rs_or_single_client(event_listeners=[listener]) self.addCleanup(client.close) coll = client[self.db.name].test_close_kills_cursors # Add some test data. docs_inserted = 1000 coll.insert_many([{"i": i} for i in range(docs_inserted)]) results.clear() # Close a cursor while it's still open on the server. cursor = coll.find().batch_size(10) self.assertTrue(bool(next(cursor))) self.assertLess(cursor.retrieved, docs_inserted) cursor.close() def assertCursorKilled(): self.assertEqual(1, len(results["started"])) self.assertEqual("killCursors", results["started"][0].command_name) self.assertEqual(1, len(results["succeeded"])) self.assertEqual("killCursors", results["succeeded"][0].command_name) assertCursorKilled() results.clear() # Close a command cursor while it's still open on the server. cursor = coll.aggregate([], batchSize=10) self.assertTrue(bool(next(cursor))) cursor.close() # The cursor should be killed if it had a non-zero id. if cursor.cursor_id: assertCursorKilled() else: self.assertEqual(0, len(results["started"])) class TestRawBatchCursor(IntegrationTest): def test_find_raw(self): c = self.db.test c.drop() docs = [{'_id': i, 'x': 3.0 * i} for i in range(10)] c.insert_many(docs) batches = list(c.find_raw_batches().sort('_id')) self.assertEqual(1, len(batches)) self.assertEqual(docs, decode_all(batches[0])) def test_manipulate(self): c = self.db.test with self.assertRaises(InvalidOperation): c.find_raw_batches(manipulate=True) def test_explain(self): c = self.db.test c.insert_one({}) explanation = c.find_raw_batches().explain() self.assertIsInstance(explanation, dict) def test_clone(self): cursor = self.db.test.find_raw_batches() # Copy of a RawBatchCursor is also a RawBatchCursor, not a Cursor. self.assertIsInstance(next(cursor.clone()), bytes) self.assertIsInstance(next(copy.copy(cursor)), bytes) @client_context.require_no_mongos def test_exhaust(self): c = self.db.test c.drop() c.insert_many({'_id': i} for i in range(200)) result = b''.join(c.find_raw_batches(cursor_type=CursorType.EXHAUST)) self.assertEqual([{'_id': i} for i in range(200)], decode_all(result)) def test_server_error(self): with self.assertRaises(OperationFailure) as exc: next(self.db.test.find_raw_batches({'x': {'$bad': 1}})) # The server response was decoded, not left raw. self.assertIsInstance(exc.exception.details, dict) def test_get_item(self): with self.assertRaises(InvalidOperation): self.db.test.find_raw_batches()[0] @client_context.require_version_min(3, 4) def test_collation(self): next(self.db.test.find_raw_batches(collation=Collation('en_US'))) @client_context.require_version_max(3, 2) def test_collation_error(self): with self.assertRaises(ConfigurationError): next(self.db.test.find_raw_batches(collation=Collation('en_US'))) @client_context.require_version_min(3, 2) def test_read_concern(self): c = self.db.get_collection("test", read_concern=ReadConcern("majority")) next(c.find_raw_batches()) @client_context.require_version_max(3, 1) def test_read_concern_error(self): c = self.db.get_collection("test", read_concern=ReadConcern("majority")) with self.assertRaises(ConfigurationError): next(c.find_raw_batches()) def test_monitoring(self): listener = EventListener() client = rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test c.drop() c.insert_many([{'_id': i} for i in range(10)]) listener.results.clear() cursor = c.find_raw_batches(batch_size=4) # First raw batch of 4 documents. next(cursor) started = listener.results['started'][0] succeeded = listener.results['succeeded'][0] self.assertEqual(0, len(listener.results['failed'])) self.assertEqual('find', started.command_name) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('find', succeeded.command_name) csr = succeeded.reply["cursor"] self.assertEqual(csr["ns"], "pymongo_test.test") # The batch is a list of one raw bytes object. self.assertEqual(len(csr["firstBatch"]), 1) self.assertEqual(decode_all(csr["firstBatch"][0]), [{'_id': i} for i in range(0, 4)]) listener.results.clear() # Next raw batch of 4 documents. next(cursor) try: results = listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertEqual('getMore', started.command_name) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('getMore', succeeded.command_name) csr = succeeded.reply["cursor"] self.assertEqual(csr["ns"], "pymongo_test.test") self.assertEqual(len(csr["nextBatch"]), 1) self.assertEqual(decode_all(csr["nextBatch"][0]), [{'_id': i} for i in range(4, 8)]) finally: # Finish the cursor. tuple(cursor) class TestRawBatchCommandCursor(IntegrationTest): @classmethod def setUpClass(cls): super(TestRawBatchCommandCursor, cls).setUpClass() def test_aggregate_raw(self): c = self.db.test c.drop() docs = [{'_id': i, 'x': 3.0 * i} for i in range(10)] c.insert_many(docs) batches = list(c.aggregate_raw_batches([{'$sort': {'_id': 1}}])) self.assertEqual(1, len(batches)) self.assertEqual(docs, decode_all(batches[0])) def test_server_error(self): c = self.db.test c.drop() docs = [{'_id': i, 'x': 3.0 * i} for i in range(10)] c.insert_many(docs) c.insert_one({'_id': 10, 'x': 'not a number'}) with self.assertRaises(OperationFailure) as exc: list(self.db.test.aggregate_raw_batches([{ '$sort': {'_id': 1}, }, { '$project': {'x': {'$multiply': [2, '$x']}} }], batchSize=4)) # The server response was decoded, not left raw. self.assertIsInstance(exc.exception.details, dict) def test_get_item(self): with self.assertRaises(InvalidOperation): self.db.test.aggregate_raw_batches([])[0] @client_context.require_version_min(3, 4) def test_collation(self): next(self.db.test.aggregate_raw_batches([], collation=Collation('en_US'))) @client_context.require_version_max(3, 2) def test_collation_error(self): with self.assertRaises(ConfigurationError): next(self.db.test.aggregate_raw_batches([], collation=Collation('en_US'))) def test_monitoring(self): listener = EventListener() client = rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test c.drop() c.insert_many([{'_id': i} for i in range(10)]) listener.results.clear() cursor = c.aggregate_raw_batches([{'$sort': {'_id': 1}}], batchSize=4) # Start cursor, no initial batch. started = listener.results['started'][0] succeeded = listener.results['succeeded'][0] self.assertEqual(0, len(listener.results['failed'])) self.assertEqual('aggregate', started.command_name) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('aggregate', succeeded.command_name) csr = succeeded.reply["cursor"] self.assertEqual(csr["ns"], "pymongo_test.test") # First batch is empty. self.assertEqual(len(csr["firstBatch"]), 0) listener.results.clear() # Batches of 4 documents. n = 0 for batch in cursor: results = listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertEqual('getMore', started.command_name) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('getMore', succeeded.command_name) csr = succeeded.reply["cursor"] self.assertEqual(csr["ns"], "pymongo_test.test") self.assertEqual(len(csr["nextBatch"]), 1) self.assertEqual(csr["nextBatch"][0], batch) self.assertEqual(decode_all(batch), [{'_id': i} for i in range(n, min(n + 4, 10))]) n += 4 listener.results.clear() if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_bulk.py0000644000076600000240000003330613245621354016772 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the bulk API.""" import sys sys.path[0:0] = [""] from bson.objectid import ObjectId from pymongo.operations import * from pymongo.errors import (ConfigurationError, InvalidOperation, OperationFailure) from pymongo.write_concern import WriteConcern from test import (client_context, unittest, IntegrationTest) from test.utils import (remove_all_users, rs_or_single_client_noauth) class BulkTestBase(IntegrationTest): @classmethod def setUpClass(cls): super(BulkTestBase, cls).setUpClass() cls.coll = cls.db.test ismaster = client_context.client.admin.command('ismaster') cls.has_write_commands = (ismaster.get("maxWireVersion", 0) > 1) def setUp(self): super(BulkTestBase, self).setUp() self.coll.drop() def assertEqualResponse(self, expected, actual): """Compare response from bulk.execute() to expected response.""" for key, value in expected.items(): if key == 'nModified': if self.has_write_commands: self.assertEqual(value, actual['nModified']) else: # Legacy servers don't include nModified in the response. self.assertFalse('nModified' in actual) elif key == 'upserted': expected_upserts = value actual_upserts = actual['upserted'] self.assertEqual( len(expected_upserts), len(actual_upserts), 'Expected %d elements in "upserted", got %d' % ( len(expected_upserts), len(actual_upserts))) for e, a in zip(expected_upserts, actual_upserts): self.assertEqualUpsert(e, a) elif key == 'writeErrors': expected_errors = value actual_errors = actual['writeErrors'] self.assertEqual( len(expected_errors), len(actual_errors), 'Expected %d elements in "writeErrors", got %d' % ( len(expected_errors), len(actual_errors))) for e, a in zip(expected_errors, actual_errors): self.assertEqualWriteError(e, a) else: self.assertEqual( actual.get(key), value, '%r value of %r does not match expected %r' % (key, actual.get(key), value)) def assertEqualUpsert(self, expected, actual): """Compare bulk.execute()['upserts'] to expected value. Like: {'index': 0, '_id': ObjectId()} """ self.assertEqual(expected['index'], actual['index']) if expected['_id'] == '...': # Unspecified value. self.assertTrue('_id' in actual) else: self.assertEqual(expected['_id'], actual['_id']) def assertEqualWriteError(self, expected, actual): """Compare bulk.execute()['writeErrors'] to expected value. Like: {'index': 0, 'code': 123, 'errmsg': '...', 'op': { ... }} """ self.assertEqual(expected['index'], actual['index']) self.assertEqual(expected['code'], actual['code']) if expected['errmsg'] == '...': # Unspecified value. self.assertTrue('errmsg' in actual) else: self.assertEqual(expected['errmsg'], actual['errmsg']) expected_op = expected['op'].copy() actual_op = actual['op'].copy() if expected_op.get('_id') == '...': # Unspecified _id. self.assertTrue('_id' in actual_op) actual_op.pop('_id') expected_op.pop('_id') self.assertEqual(expected_op, actual_op) class TestBulk(BulkTestBase): def test_empty(self): self.assertRaises(InvalidOperation, self.coll.bulk_write, []) def test_insert(self): expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 1, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } result = self.coll.bulk_write([InsertOne({})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.inserted_count) self.assertEqual(1, self.coll.count()) def test_update_many(self): expected = { 'nMatched': 2, 'nModified': 2, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) result = self.coll.bulk_write([UpdateMany({}, {'$set': {'foo': 'bar'}})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(2, result.matched_count) self.assertTrue(result.modified_count in (2, None)) @client_context.require_version_max(3, 5, 5) def test_array_filters_unsupported(self): requests = [ UpdateMany( {}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]), UpdateOne( {}, {'$set': {"y.$[i].b": 2}}, array_filters=[{'i.b': 3}]) ] for bulk_op in requests: self.assertRaises( ConfigurationError, self.coll.bulk_write, [bulk_op]) def test_array_filters_validation(self): self.assertRaises(TypeError, UpdateMany, {}, {}, array_filters={}) self.assertRaises(TypeError, UpdateOne, {}, {}, array_filters={}) def test_array_filters_unacknowledged(self): coll = self.coll.with_options(write_concern=WriteConcern(w=0)) update_one = UpdateOne( {}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]) update_many = UpdateMany( {}, {'$set': {'y.$[i].b': 5}}, array_filters=[{'i.b': 1}]) self.assertRaises(ConfigurationError, coll.bulk_write, [update_one]) self.assertRaises(ConfigurationError, coll.bulk_write, [update_many]) def test_update_one(self): expected = { 'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) result = self.coll.bulk_write([UpdateOne({}, {'$set': {'foo': 'bar'}})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (1, None)) def test_replace_one(self): expected = { 'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) result = self.coll.bulk_write([ReplaceOne({}, {'foo': 'bar'})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (1, None)) def test_remove(self): # Test removing all documents, ordered. expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) result = self.coll.bulk_write([DeleteMany({})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(2, result.deleted_count) def test_remove_one(self): # Test removing one document, empty selector. self.coll.insert_many([{}, {}]) expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 1, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } result = self.coll.bulk_write([DeleteOne({})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.deleted_count) self.assertEqual(self.coll.count(), 1) def test_upsert(self): expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 1, 'nInserted': 0, 'nRemoved': 0, 'upserted': [{'index': 0, '_id': '...'}] } result = self.coll.bulk_write([ReplaceOne({}, {'foo': 'bar'}, upsert=True)]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.upserted_count) self.assertEqual(1, len(result.upserted_ids)) self.assertTrue(isinstance(result.upserted_ids.get(0), ObjectId)) self.assertEqual(self.coll.find({'foo': 'bar'}).count(), 1) def test_numerous_inserts(self): # Ensure we don't exceed server's 1000-document batch size limit. n_docs = 2100 requests = [InsertOne({}) for _ in range(n_docs)] result = self.coll.bulk_write(requests, ordered=False) self.assertEqual(n_docs, result.inserted_count) self.assertEqual(n_docs, self.coll.count()) # Same with ordered bulk. self.coll.drop() result = self.coll.bulk_write(requests) self.assertEqual(n_docs, result.inserted_count) self.assertEqual(n_docs, self.coll.count()) def test_generator_insert(self): def gen(): yield {'a': 1, 'b': 1} yield {'a': 1, 'b': 2} yield {'a': 2, 'b': 3} yield {'a': 3, 'b': 5} yield {'a': 5, 'b': 8} result = self.coll.insert_many(gen()) self.assertEqual(5, len(result.inserted_ids)) def test_bulk_write_no_results(self): coll = self.coll.with_options(write_concern=WriteConcern(w=0)) result = coll.bulk_write([InsertOne({})]) self.assertFalse(result.acknowledged) self.assertRaises(InvalidOperation, lambda: result.inserted_count) self.assertRaises(InvalidOperation, lambda: result.matched_count) self.assertRaises(InvalidOperation, lambda: result.modified_count) self.assertRaises(InvalidOperation, lambda: result.deleted_count) self.assertRaises(InvalidOperation, lambda: result.upserted_count) self.assertRaises(InvalidOperation, lambda: result.upserted_ids) def test_bulk_write_invalid_arguments(self): # The requests argument must be a list. generator = (InsertOne({}) for _ in range(10)) with self.assertRaises(TypeError): self.coll.bulk_write(generator) # Document is not wrapped in a bulk write operation. with self.assertRaises(TypeError): self.coll.bulk_write([{}]) class BulkAuthorizationTestBase(BulkTestBase): @classmethod @client_context.require_auth def setUpClass(cls): super(BulkAuthorizationTestBase, cls).setUpClass() def setUp(self): super(BulkAuthorizationTestBase, self).setUp() client_context.create_user( self.db.name, 'readonly', 'pw', ['read']) self.db.command( 'createRole', 'noremove', privileges=[{ 'actions': ['insert', 'update', 'find'], 'resource': {'db': 'pymongo_test', 'collection': 'test'} }], roles=[]) client_context.create_user(self.db.name, 'noremove', 'pw', ['noremove']) def tearDown(self): self.db.command('dropRole', 'noremove') remove_all_users(self.db) class TestBulkAuthorization(BulkAuthorizationTestBase): def test_readonly(self): # We test that an authorization failure aborts the batch and is raised # as OperationFailure. cli = rs_or_single_client_noauth(username='readonly', password='pw', authSource='pymongo_test') coll = cli.pymongo_test.test coll.find_one() self.assertRaises(OperationFailure, coll.bulk_write, [InsertOne({'x': 1})]) def test_no_remove(self): # We test that an authorization failure aborts the batch and is raised # as OperationFailure. cli = rs_or_single_client_noauth(username='noremove', password='pw', authSource='pymongo_test') coll = cli.pymongo_test.test coll.find_one() requests = [ InsertOne({'x': 1}), ReplaceOne({'x': 2}, {'x': 2}, upsert=True), DeleteMany({}), # Prohibited. InsertOne({'x': 3}), # Never attempted. ] self.assertRaises(OperationFailure, coll.bulk_write, requests) self.assertEqual(set([1, 2]), set(self.coll.distinct('x'))) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_uri_spec.py0000644000076600000240000001111213245617773017647 0ustar shanestaff00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the pymongo uri_parser module is up to spec.""" import json import os import sys import warnings sys.path[0:0] = [""] from pymongo.uri_parser import parse_uri from test import unittest # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), os.path.join('connection_string', 'test')) class TestAllScenarios(unittest.TestCase): pass def create_test(scenario_def): def run_scenario(self): self.assertTrue(scenario_def['tests'], "tests cannot be empty") for test in scenario_def['tests']: dsc = test['description'] warned = False error = False with warnings.catch_warnings(): warnings.filterwarnings('error') try: options = parse_uri(test['uri'], warn=True) except Warning: warned = True except Exception: error = True self.assertEqual(not error, test['valid'], "Test failure '%s'" % dsc) if test.get("warning", False): self.assertTrue(warned, "Expected warning for test '%s'" % (dsc,)) # Redo in the case there were warnings that were not expected. if warned: options = parse_uri(test['uri'], warn=True) # Compare hosts and port. if test['hosts'] is not None: self.assertEqual( len(test['hosts']), len(options['nodelist']), "Incorrect number of hosts parsed from URI") for exp, actual in zip(test['hosts'], options['nodelist']): self.assertEqual(exp['host'], actual[0], "Expected host %s but got %s" % (exp['host'], actual[0])) if exp['port'] is not None: self.assertEqual(exp['port'], actual[1], "Expected port %s but got %s" % (exp['port'], actual)) # Compare auth options. auth = test['auth'] if auth is not None: auth['database'] = auth.pop('db') # db == database # Special case for PyMongo's collection parsing. if options.get('collection') is not None: options['database'] += "." + options['collection'] for elm in auth: if auth[elm] is not None: self.assertEqual(auth[elm], options[elm], "Expected %s but got %s" % (auth[elm], options[elm])) # Compare URI options. if test['options'] is not None: for opt in test['options']: if options.get(opt) is not None: self.assertEqual( options[opt], test['options'][opt], "For option %s expected %s but got %s" % (opt, options[opt], test['options'][opt])) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): dirname = os.path.split(dirpath) dirname = os.path.split(dirname[-2])[-1] + '_' + dirname[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json.load(scenario_stream) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s_%s' % ( dirname, os.path.splitext(filename)[0]) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_change_stream.py0000644000076600000240000004067513245621354020644 0ustar shanestaff00000000000000# Copyright 2017 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the change_stream module.""" import random import sys import string import threading import time import uuid sys.path[0:0] = [''] from bson import BSON, ObjectId, SON from bson.binary import (ALL_UUID_REPRESENTATIONS, Binary, STANDARD, PYTHON_LEGACY) from bson.raw_bson import DEFAULT_RAW_BSON_OPTIONS, RawBSONDocument from pymongo.command_cursor import CommandCursor from pymongo.errors import (InvalidOperation, OperationFailure, ServerSelectionTimeoutError) from pymongo.message import _CursorAddress from pymongo.read_concern import ReadConcern from test import client_context, unittest, IntegrationTest from test.utils import WhiteListEventListener, rs_or_single_client class TestChangeStream(IntegrationTest): @classmethod @client_context.require_version_min(3, 5, 11) @client_context.require_no_standalone def setUpClass(cls): super(TestChangeStream, cls).setUpClass() cls.coll = cls.db.change_stream_test # SERVER-31885 On a mongos the database must exist in order to create # a changeStream cursor. However, WiredTiger drops the database when # there are no more collections. Let's prevent that. cls.db.prevent_implicit_database_deletion.insert_one({}) @classmethod def tearDownClass(cls): cls.db.prevent_implicit_database_deletion.drop() super(TestChangeStream, cls).tearDownClass() def setUp(self): # Use a new collection for each test. self.coll = self.db[self.id()] def tearDown(self): self.coll.drop() def insert_and_check(self, change_stream, doc): self.coll.insert_one(doc) change = next(change_stream) self.assertEqual(change['operationType'], 'insert') self.assertEqual(change['ns'], {'db': self.coll.database.name, 'coll': self.coll.name}) self.assertEqual(change['fullDocument'], doc) def test_watch(self): with self.coll.watch( [{'$project': {'foo': 0}}], full_document='updateLookup', max_await_time_ms=1000, batch_size=100) as change_stream: self.assertEqual([{'$project': {'foo': 0}}], change_stream._pipeline) self.assertEqual('updateLookup', change_stream._full_document) self.assertIsNone(change_stream._resume_token) self.assertEqual(1000, change_stream._max_await_time_ms) self.assertEqual(100, change_stream._batch_size) self.assertIsInstance(change_stream._cursor, CommandCursor) self.assertEqual( 1000, change_stream._cursor._CommandCursor__max_await_time_ms) self.coll.insert_one({}) change = change_stream.next() resume_token = change['_id'] with self.assertRaises(TypeError): self.coll.watch(pipeline={}) with self.assertRaises(TypeError): self.coll.watch(full_document={}) # No Error. with self.coll.watch(resume_after=resume_token): pass def test_full_pipeline(self): """$changeStream must be the first stage in a change stream pipeline sent to the server. """ listener = WhiteListEventListener("aggregate") results = listener.results client = rs_or_single_client(event_listeners=[listener]) self.addCleanup(client.close) coll = client[self.db.name][self.coll.name] with coll.watch([{'$project': {'foo': 0}}]) as change_stream: self.assertEqual([{'$changeStream': {'fullDocument': 'default'}}, {'$project': {'foo': 0}}], change_stream._full_pipeline()) self.assertEqual(1, len(results['started'])) command = results['started'][0] self.assertEqual('aggregate', command.command_name) self.assertEqual([{'$changeStream': {'fullDocument': 'default'}}, {'$project': {'foo': 0}}], command.command['pipeline']) def test_iteration(self): with self.coll.watch(batch_size=2) as change_stream: num_inserted = 10 self.coll.insert_many([{} for _ in range(num_inserted)]) self.coll.drop() received = 0 for change in change_stream: received += 1 if change['operationType'] != 'invalidate': self.assertEqual(change['operationType'], 'insert') self.assertEqual(num_inserted + 1, received) with self.assertRaises(StopIteration): change_stream.next() with self.assertRaises(StopIteration): next(change_stream) def test_next_blocks(self): """Test that next blocks until a change is readable""" inserted_doc = {'_id': ObjectId()} # Use a short await time to speed up the test. with self.coll.watch(max_await_time_ms=250) as change_stream: changes = [] t = threading.Thread( target=lambda: changes.append(change_stream.next())) t.start() self.coll.insert_one(inserted_doc) time.sleep(1) t.join(3) self.assertFalse(t.is_alive()) self.assertEqual(1, len(changes)) self.assertEqual(changes[0]['operationType'], 'insert') self.assertEqual(changes[0]['fullDocument'], inserted_doc) def test_aggregate_cursor_blocks(self): """Test that an aggregate cursor blocks until a change is readable.""" inserted_doc = {'_id': ObjectId()} with self.coll.aggregate([{'$changeStream': {}}], maxAwaitTimeMS=250) as change_stream: changes = [] t = threading.Thread( target=lambda: changes.append(change_stream.next())) t.start() self.coll.insert_one(inserted_doc) time.sleep(1) t.join(3) self.assertFalse(t.is_alive()) self.assertEqual(1, len(changes)) self.assertEqual(changes[0]['operationType'], 'insert') self.assertEqual(changes[0]['fullDocument'], inserted_doc) def test_concurrent_close(self): """Ensure a ChangeStream can be closed from another thread.""" # Use a short await time to speed up the test. with self.coll.watch(max_await_time_ms=250) as change_stream: def iterate_cursor(): for change in change_stream: pass t = threading.Thread(target=iterate_cursor) t.start() self.coll.insert_one({}) time.sleep(1) change_stream.close() t.join(3) self.assertFalse(t.is_alive()) def test_update_resume_token(self): """ChangeStream must continuously track the last seen resumeToken.""" with self.coll.watch() as change_stream: self.assertIsNone(change_stream._resume_token) for i in range(10): self.coll.insert_one({}) change = next(change_stream) self.assertEqual(change['_id'], change_stream._resume_token) def test_raises_error_on_missing_id(self): """ChangeStream will raise an exception if the server response is missing the resume token. """ with self.coll.watch([{'$project': {'_id': 0}}]) as change_stream: self.coll.insert_one({}) with self.assertRaises(InvalidOperation): next(change_stream) # The cursor should now be closed. with self.assertRaises(StopIteration): next(change_stream) def test_resume_on_error(self): """ChangeStream will automatically resume one time on a resumable error (including not master) with the initial pipeline and options, except for the addition/update of a resumeToken. """ with self.coll.watch([]) as change_stream: self.insert_and_check(change_stream, {'_id': 1}) # Cause a cursor not found error on the next getMore. cursor = change_stream._cursor address = _CursorAddress(cursor.address, self.coll.full_name) self.client._close_cursor_now(cursor.cursor_id, address) self.insert_and_check(change_stream, {'_id': 2}) def test_does_not_resume_on_server_error(self): """ChangeStream will not attempt to resume on a server error.""" def mock_next(self, *args, **kwargs): self._CommandCursor__killed = True raise OperationFailure('Mock server error') original_next = CommandCursor.next CommandCursor.next = mock_next try: with self.coll.watch() as change_stream: with self.assertRaises(OperationFailure): next(change_stream) CommandCursor.next = original_next with self.assertRaises(StopIteration): next(change_stream) finally: CommandCursor.next = original_next def test_initial_empty_batch(self): """Ensure that a cursor returned from an aggregate command with a cursor id, and an initial empty batch, is not closed on the driver side. """ with self.coll.watch() as change_stream: # The first batch should be empty. self.assertEqual( 0, len(change_stream._cursor._CommandCursor__data)) cursor_id = change_stream._cursor.cursor_id self.assertTrue(cursor_id) self.insert_and_check(change_stream, {}) # Make sure we're still using the same cursor. self.assertEqual(cursor_id, change_stream._cursor.cursor_id) def test_kill_cursors(self): """The killCursors command sent during the resume process must not be allowed to raise an exception. """ def raise_error(): raise ServerSelectionTimeoutError('mock error') with self.coll.watch([]) as change_stream: self.insert_and_check(change_stream, {'_id': 1}) # Cause a cursor not found error on the next getMore. cursor = change_stream._cursor address = _CursorAddress(cursor.address, self.coll.full_name) self.client._close_cursor_now(cursor.cursor_id, address) cursor.close = raise_error self.insert_and_check(change_stream, {'_id': 2}) def test_unknown_full_document(self): """Must rely on the server to raise an error on unknown fullDocument. """ try: with self.coll.watch(full_document='notValidatedByPyMongo'): pass except OperationFailure: pass def test_change_operations(self): """Test each operation type.""" expected_ns = {'db': self.coll.database.name, 'coll': self.coll.name} with self.coll.watch() as change_stream: # Insert. inserted_doc = {'_id': ObjectId(), 'foo': 'bar'} self.coll.insert_one(inserted_doc) change = change_stream.next() self.assertTrue(change['_id']) self.assertEqual(change['operationType'], 'insert') self.assertEqual(change['ns'], expected_ns) self.assertEqual(change['fullDocument'], inserted_doc) # Update. update_spec = {'$set': {'new': 1}, '$unset': {'foo': 1}} self.coll.update_one(inserted_doc, update_spec) change = change_stream.next() self.assertTrue(change['_id']) self.assertEqual(change['operationType'], 'update') self.assertEqual(change['ns'], expected_ns) self.assertNotIn('fullDocument', change) self.assertEqual({'updatedFields': {'new': 1}, 'removedFields': ['foo']}, change['updateDescription']) # Replace. self.coll.replace_one({'new': 1}, {'foo': 'bar'}) change = change_stream.next() self.assertTrue(change['_id']) self.assertEqual(change['operationType'], 'replace') self.assertEqual(change['ns'], expected_ns) self.assertEqual(change['fullDocument'], inserted_doc) # Delete. self.coll.delete_one({'foo': 'bar'}) change = change_stream.next() self.assertTrue(change['_id']) self.assertEqual(change['operationType'], 'delete') self.assertEqual(change['ns'], expected_ns) self.assertNotIn('fullDocument', change) # Invalidate. self.coll.drop() change = change_stream.next() self.assertTrue(change['_id']) self.assertEqual(change['operationType'], 'invalidate') self.assertNotIn('ns', change) self.assertNotIn('fullDocument', change) # The ChangeStream should be dead. with self.assertRaises(StopIteration): change_stream.next() def test_raw(self): """Test with RawBSONDocument.""" raw_coll = self.coll.with_options( codec_options=DEFAULT_RAW_BSON_OPTIONS) with raw_coll.watch() as change_stream: raw_doc = RawBSONDocument(BSON.encode({'_id': 1})) self.coll.insert_one(raw_doc) change = next(change_stream) self.assertIsInstance(change, RawBSONDocument) self.assertEqual(change['operationType'], 'insert') self.assertEqual(change['ns']['db'], self.coll.database.name) self.assertEqual(change['ns']['coll'], self.coll.name) self.assertEqual(change['fullDocument'], raw_doc) self.assertEqual(change['_id'], change_stream._resume_token) def test_uuid_representations(self): """Test with uuid document _ids and different uuid_representation.""" for uuid_representation in ALL_UUID_REPRESENTATIONS: for id_subtype in (STANDARD, PYTHON_LEGACY): resume_token = None options = self.coll.codec_options.with_options( uuid_representation=uuid_representation) coll = self.coll.with_options(codec_options=options) with coll.watch() as change_stream: coll.insert_one( {'_id': Binary(uuid.uuid4().bytes, id_subtype)}) resume_token = change_stream.next()['_id'] # Should not error. coll.watch(resume_after=resume_token) def test_document_id_order(self): """Test with document _ids that need their order preserved.""" random_keys = random.sample(string.ascii_letters, len(string.ascii_letters)) random_doc = {'_id': SON([(key, key) for key in random_keys])} for document_class in (dict, SON, RawBSONDocument): options = self.coll.codec_options.with_options( document_class=document_class) coll = self.coll.with_options(codec_options=options) with coll.watch() as change_stream: coll.insert_one(random_doc) resume_token = change_stream.next()['_id'] # The resume token is always a document. self.assertIsInstance(resume_token, document_class) # Should not error. coll.watch(resume_after=resume_token) coll.delete_many({}) def test_read_concern(self): """Test readConcern is not validated by the driver.""" # Read concern 'local' is not allowed for $changeStream. coll = self.coll.with_options(read_concern=ReadConcern('local')) with self.assertRaises(OperationFailure): coll.watch() # Does not error. coll = self.coll.with_options(read_concern=ReadConcern('majority')) with coll.watch(): pass if __name__ == '__main__': unittest.main() pymongo-3.6.1/test/test_server_description.py0000644000076600000240000001273013245621354021744 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the server_description module.""" import sys sys.path[0:0] = [""] from pymongo.server_type import SERVER_TYPE from pymongo.ismaster import IsMaster from pymongo.server_description import ServerDescription from test import unittest address = ('localhost', 27017) def parse_ismaster_response(doc): ismaster_response = IsMaster(doc) return ServerDescription(address, ismaster_response) class TestServerDescription(unittest.TestCase): def test_unknown(self): # Default, no ismaster_response. s = ServerDescription(address) self.assertEqual(SERVER_TYPE.Unknown, s.server_type) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_mongos(self): s = parse_ismaster_response({'ok': 1, 'msg': 'isdbgrid'}) self.assertEqual(SERVER_TYPE.Mongos, s.server_type) self.assertEqual('Mongos', s.server_type_name) self.assertTrue(s.is_writable) self.assertTrue(s.is_readable) def test_primary(self): s = parse_ismaster_response( {'ok': 1, 'ismaster': True, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSPrimary, s.server_type) self.assertEqual('RSPrimary', s.server_type_name) self.assertTrue(s.is_writable) self.assertTrue(s.is_readable) def test_secondary(self): s = parse_ismaster_response( {'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSSecondary, s.server_type) self.assertEqual('RSSecondary', s.server_type_name) self.assertFalse(s.is_writable) self.assertTrue(s.is_readable) def test_arbiter(self): s = parse_ismaster_response( {'ok': 1, 'ismaster': False, 'arbiterOnly': True, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSArbiter, s.server_type) self.assertEqual('RSArbiter', s.server_type_name) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_other(self): s = parse_ismaster_response( {'ok': 1, 'ismaster': False, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSOther, s.server_type) self.assertEqual('RSOther', s.server_type_name) s = parse_ismaster_response({ 'ok': 1, 'ismaster': False, 'secondary': True, 'hidden': True, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSOther, s.server_type) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_ghost(self): s = parse_ismaster_response({'ok': 1, 'isreplicaset': True}) self.assertEqual(SERVER_TYPE.RSGhost, s.server_type) self.assertEqual('RSGhost', s.server_type_name) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_fields(self): s = parse_ismaster_response({ 'ok': 1, 'ismaster': False, 'secondary': True, 'primary': 'a:27017', 'tags': {'a': 'foo', 'b': 'baz'}, 'maxMessageSizeBytes': 1, 'maxBsonObjectSize': 2, 'maxWriteBatchSize': 3, 'minWireVersion': 4, 'maxWireVersion': 5, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSSecondary, s.server_type) self.assertEqual(('a', 27017), s.primary) self.assertEqual({'a': 'foo', 'b': 'baz'}, s.tags) self.assertEqual(1, s.max_message_size) self.assertEqual(2, s.max_bson_size) self.assertEqual(3, s.max_write_batch_size) self.assertEqual(4, s.min_wire_version) self.assertEqual(5, s.max_wire_version) def test_default_max_message_size(self): s = parse_ismaster_response({ 'ok': 1, 'ismaster': True, 'maxBsonObjectSize': 2}) # Twice max_bson_size. self.assertEqual(4, s.max_message_size) def test_standalone(self): s = parse_ismaster_response({'ok': 1, 'ismaster': True}) self.assertEqual(SERVER_TYPE.Standalone, s.server_type) # Mongod started with --slave. s = parse_ismaster_response({'ok': 1, 'ismaster': False}) self.assertEqual(SERVER_TYPE.Standalone, s.server_type) self.assertTrue(s.is_writable) self.assertTrue(s.is_readable) def test_ok_false(self): s = parse_ismaster_response({'ok': 0, 'ismaster': True}) self.assertEqual(SERVER_TYPE.Unknown, s.server_type) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_all_hosts(self): s = parse_ismaster_response({ 'ok': 1, 'ismaster': True, 'hosts': ['a'], 'passives': ['b:27018'], 'arbiters': ['c'] }) self.assertEqual( [('a', 27017), ('b', 27018), ('c', 27017)], sorted(s.all_hosts)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_son_manipulator.py0000644000076600000240000001046213245621354021245 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for SONManipulators. """ import sys import warnings sys.path[0:0] = [""] from bson.son import SON from pymongo import MongoClient from pymongo.son_manipulator import (NamespaceInjector, ObjectIdInjector, ObjectIdShuffler, SONManipulator) from test import client_context, qcheck, unittest class TestSONManipulator(unittest.TestCase): @classmethod def setUpClass(cls): cls.warn_context = warnings.catch_warnings() cls.warn_context.__enter__() warnings.simplefilter("ignore", DeprecationWarning) client = MongoClient( client_context.host, client_context.port, connect=False) cls.db = client.pymongo_test @classmethod def tearDownClass(cls): cls.warn_context.__exit__() cls.warn_context = None def test_basic(self): manip = SONManipulator() collection = self.db.test def incoming_is_identity(son): return son == manip.transform_incoming(son, collection) qcheck.check_unittest(self, incoming_is_identity, qcheck.gen_mongo_dict(3)) def outgoing_is_identity(son): return son == manip.transform_outgoing(son, collection) qcheck.check_unittest(self, outgoing_is_identity, qcheck.gen_mongo_dict(3)) def test_id_injection(self): manip = ObjectIdInjector() collection = self.db.test def incoming_adds_id(son): son = manip.transform_incoming(son, collection) assert "_id" in son return True qcheck.check_unittest(self, incoming_adds_id, qcheck.gen_mongo_dict(3)) def outgoing_is_identity(son): return son == manip.transform_outgoing(son, collection) qcheck.check_unittest(self, outgoing_is_identity, qcheck.gen_mongo_dict(3)) def test_id_shuffling(self): manip = ObjectIdShuffler() collection = self.db.test def incoming_moves_id(son_in): son = manip.transform_incoming(son_in, collection) if not "_id" in son: return True for (k, v) in son.items(): self.assertEqual(k, "_id") break # Key order matters in SON equality test, # matching collections.OrderedDict if isinstance(son_in, SON): return son_in.to_dict() == son.to_dict() return son_in == son self.assertTrue(incoming_moves_id({})) self.assertTrue(incoming_moves_id({"_id": 12})) self.assertTrue(incoming_moves_id({"hello": "world", "_id": 12})) self.assertTrue(incoming_moves_id(SON([("hello", "world"), ("_id", 12)]))) def outgoing_is_identity(son): return son == manip.transform_outgoing(son, collection) qcheck.check_unittest(self, outgoing_is_identity, qcheck.gen_mongo_dict(3)) def test_ns_injection(self): manip = NamespaceInjector() collection = self.db.test def incoming_adds_ns(son): son = manip.transform_incoming(son, collection) assert "_ns" in son return son["_ns"] == collection.name qcheck.check_unittest(self, incoming_adds_ns, qcheck.gen_mongo_dict(3)) def outgoing_is_identity(son): return son == manip.transform_outgoing(son, collection) qcheck.check_unittest(self, outgoing_is_identity, qcheck.gen_mongo_dict(3)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_son.py0000644000076600000240000001571213245621354016635 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the son module.""" import copy import pickle import re import sys sys.path[0:0] = [""] from bson.py3compat import b from bson.son import SON from test import SkipTest, unittest class TestSON(unittest.TestCase): def test_ordered_dict(self): a1 = SON() a1["hello"] = "world" a1["mike"] = "awesome" a1["hello_"] = "mike" self.assertEqual(list(a1.items()), [("hello", "world"), ("mike", "awesome"), ("hello_", "mike")]) b2 = SON({"hello": "world"}) self.assertEqual(b2["hello"], "world") self.assertRaises(KeyError, lambda: b2["goodbye"]) def test_equality(self): a1 = SON({"hello": "world"}) b2 = SON((('hello', 'world'), ('mike', 'awesome'), ('hello_', 'mike'))) self.assertEqual(a1, SON({"hello": "world"})) self.assertEqual(b2, SON((('hello', 'world'), ('mike', 'awesome'), ('hello_', 'mike')))) self.assertEqual(b2, dict((('hello_', 'mike'), ('mike', 'awesome'), ('hello', 'world')))) self.assertNotEqual(a1, b2) self.assertNotEqual(b2, SON((('hello_', 'mike'), ('mike', 'awesome'), ('hello', 'world')))) # Explicitly test inequality self.assertFalse(a1 != SON({"hello": "world"})) self.assertFalse(b2 != SON((('hello', 'world'), ('mike', 'awesome'), ('hello_', 'mike')))) self.assertFalse(b2 != dict((('hello_', 'mike'), ('mike', 'awesome'), ('hello', 'world')))) # Embedded SON. d4 = SON([('blah', {'foo': SON()})]) self.assertEqual(d4, {'blah': {'foo': {}}}) self.assertEqual(d4, {'blah': {'foo': SON()}}) self.assertNotEqual(d4, {'blah': {'foo': []}}) # Original data unaffected. self.assertEqual(SON, d4['blah']['foo'].__class__) def test_to_dict(self): a1 = SON() b2 = SON([("blah", SON())]) c3 = SON([("blah", [SON()])]) d4 = SON([("blah", {"foo": SON()})]) self.assertEqual({}, a1.to_dict()) self.assertEqual({"blah": {}}, b2.to_dict()) self.assertEqual({"blah": [{}]}, c3.to_dict()) self.assertEqual({"blah": {"foo": {}}}, d4.to_dict()) self.assertEqual(dict, a1.to_dict().__class__) self.assertEqual(dict, b2.to_dict()["blah"].__class__) self.assertEqual(dict, c3.to_dict()["blah"][0].__class__) self.assertEqual(dict, d4.to_dict()["blah"]["foo"].__class__) # Original data unaffected. self.assertEqual(SON, d4['blah']['foo'].__class__) def test_pickle(self): simple_son = SON([]) complex_son = SON([('son', simple_son), ('list', [simple_son, simple_son])]) for protocol in range(pickle.HIGHEST_PROTOCOL + 1): pickled = pickle.loads(pickle.dumps(complex_son, protocol=protocol)) self.assertEqual(pickled['son'], pickled['list'][0]) self.assertEqual(pickled['son'], pickled['list'][1]) def test_pickle_backwards_compatability(self): # This string was generated by pickling a SON object in pymongo # version 2.1.1 pickled_with_2_1_1 = b( "ccopy_reg\n_reconstructor\np0\n(cbson.son\nSON\np1\n" "c__builtin__\ndict\np2\n(dp3\ntp4\nRp5\n(dp6\n" "S'_SON__keys'\np7\n(lp8\nsb." ) son_2_1_1 = pickle.loads(pickled_with_2_1_1) self.assertEqual(son_2_1_1, SON([])) def test_copying(self): simple_son = SON([]) complex_son = SON([('son', simple_son), ('list', [simple_son, simple_son])]) regex_son = SON([("x", re.compile("^hello.*"))]) reflexive_son = SON([('son', simple_son)]) reflexive_son["reflexive"] = reflexive_son simple_son1 = copy.copy(simple_son) self.assertEqual(simple_son, simple_son1) complex_son1 = copy.copy(complex_son) self.assertEqual(complex_son, complex_son1) regex_son1 = copy.copy(regex_son) self.assertEqual(regex_son, regex_son1) reflexive_son1 = copy.copy(reflexive_son) self.assertEqual(reflexive_son, reflexive_son1) # Test deepcopying simple_son1 = copy.deepcopy(simple_son) self.assertEqual(simple_son, simple_son1) regex_son1 = copy.deepcopy(regex_son) self.assertEqual(regex_son, regex_son1) complex_son1 = copy.deepcopy(complex_son) self.assertEqual(complex_son, complex_son1) reflexive_son1 = copy.deepcopy(reflexive_son) self.assertEqual(list(reflexive_son), list(reflexive_son1)) self.assertEqual(id(reflexive_son1), id(reflexive_son1["reflexive"])) def test_iteration(self): """ Test __iter__ """ # test success case test_son = SON([(1, 100), (2, 200), (3, 300)]) for ele in test_son: self.assertEqual(ele * 100, test_son[ele]) def test_contains_has(self): """ has_key and __contains__ """ test_son = SON([(1, 100), (2, 200), (3, 300)]) self.assertIn(1, test_son) self.assertTrue(2 in test_son, "in failed") self.assertFalse(22 in test_son, "in succeeded when it shouldn't") self.assertTrue(test_son.has_key(2), "has_key failed") self.assertFalse(test_son.has_key(22), "has_key succeeded when it shouldn't") def test_clears(self): """ Test clear() """ test_son = SON([(1, 100), (2, 200), (3, 300)]) test_son.clear() self.assertNotIn(1, test_son) self.assertEqual(0, len(test_son)) self.assertEqual(0, len(test_son.keys())) self.assertEqual({}, test_son.to_dict()) def test_len(self): """ Test len """ test_son = SON() self.assertEqual(0, len(test_son)) test_son = SON([(1, 100), (2, 200), (3, 300)]) self.assertEqual(3, len(test_son)) test_son.popitem() self.assertEqual(2, len(test_son)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_monitoring.py0000644000076600000240000017754513245621354020240 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import copy import datetime import sys import time import warnings sys.path[0:0] = [""] from bson.objectid import ObjectId from bson.py3compat import text_type from bson.son import SON from pymongo import CursorType, monitoring, InsertOne, UpdateOne, DeleteOne from pymongo.command_cursor import CommandCursor from pymongo.errors import NotMasterError, OperationFailure from pymongo.read_preferences import ReadPreference from pymongo.write_concern import WriteConcern from test import (client_context, client_knobs, PyMongoTestCase, sanitize_cmd, unittest) from test.utils import (EventListener, rs_or_single_client, single_client, wait_until) class TestCommandMonitoring(PyMongoTestCase): @classmethod @client_context.require_connection def setUpClass(cls): cls.listener = EventListener() cls.saved_listeners = monitoring._LISTENERS # Don't use any global subscribers. monitoring._LISTENERS = monitoring._Listeners([], [], [], []) cls.client = rs_or_single_client(event_listeners=[cls.listener]) @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners def tearDown(self): self.listener.results.clear() def test_started_simple(self): self.client.pymongo_test.command('ismaster') results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand(SON([('ismaster', 1)]), started.command) self.assertEqual('ismaster', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) def test_succeeded_simple(self): self.client.pymongo_test.command('ismaster') results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertEqual('ismaster', succeeded.command_name) self.assertEqual(self.client.address, succeeded.connection_id) self.assertEqual(1, succeeded.reply.get('ok')) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertTrue(isinstance(succeeded.duration_micros, int)) def test_failed_simple(self): try: self.client.pymongo_test.command('oops!') except OperationFailure: pass results = self.listener.results started = results['started'][0] failed = results['failed'][0] self.assertEqual(0, len(results['succeeded'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertTrue( isinstance(failed, monitoring.CommandFailedEvent)) self.assertEqual('oops!', failed.command_name) self.assertEqual(self.client.address, failed.connection_id) self.assertEqual(0, failed.failure.get('ok')) self.assertTrue(isinstance(failed.request_id, int)) self.assertTrue(isinstance(failed.duration_micros, int)) def test_find_one(self): self.client.pymongo_test.test.find_one() results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand( SON([('find', 'test'), ('filter', {}), ('limit', 1), ('singleBatch', True)]), started.command) self.assertEqual('find', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) def test_find_and_get_more(self): self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many([{} for _ in range(10)]) self.listener.results.clear() cursor = self.client.pymongo_test.test.find( projection={'_id': False}, batch_size=4) for _ in range(4): next(cursor) cursor_id = cursor.cursor_id results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand( SON([('find', 'test'), ('filter', {}), ('projection', {'_id': False}), ('batchSize', 4)]), started.command) self.assertEqual('find', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('find', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) csr = succeeded.reply["cursor"] self.assertEqual(csr["id"], cursor_id) self.assertEqual(csr["ns"], "pymongo_test.test") self.assertEqual(csr["firstBatch"], [{} for _ in range(4)]) self.listener.results.clear() # Next batch. Exhausting the cursor could cause a getMore # that returns id of 0 and no results. next(cursor) try: results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand( SON([('getMore', cursor_id), ('collection', 'test'), ('batchSize', 4)]), started.command) self.assertEqual('getMore', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('getMore', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) csr = succeeded.reply["cursor"] self.assertEqual(csr["id"], cursor_id) self.assertEqual(csr["ns"], "pymongo_test.test") self.assertEqual(csr["nextBatch"], [{} for _ in range(4)]) finally: # Exhaust the cursor to avoid kill cursors. tuple(cursor) def test_find_with_explain(self): cmd = SON([('explain', SON([('find', 'test'), ('filter', {})]))]) self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_one({}) self.listener.results.clear() coll = self.client.pymongo_test.test # Test that we publish the unwrapped command. if self.client.is_mongos: coll = coll.with_options( read_preference=ReadPreference.PRIMARY_PREFERRED) res = coll.find().explain() results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand(cmd, started.command) self.assertEqual('explain', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('explain', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(self.client.address, succeeded.connection_id) self.assertEqual(res, succeeded.reply) def _test_find_options(self, query, expected_cmd): coll = self.client.pymongo_test.test coll.drop() coll.create_index('x') coll.insert_many([{'x': i} for i in range(5)]) # Test that we publish the unwrapped command. self.listener.results.clear() if self.client.is_mongos: coll = coll.with_options( read_preference=ReadPreference.PRIMARY_PREFERRED) cursor = coll.find(**query) next(cursor) try: results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand(expected_cmd, started.command) self.assertEqual('find', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('find', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(self.client.address, succeeded.connection_id) finally: # Exhaust the cursor to avoid kill cursors. tuple(cursor) def test_find_options(self): query = dict(filter={}, hint=[('x', 1)], max_scan=10, max_time_ms=10000, max={'x': 10}, min={'x': -10}, return_key=True, show_record_id=True, projection={'x': False}, skip=1, no_cursor_timeout=True, sort=[('_id', 1)], allow_partial_results=True, comment='this is a test', batch_size=2) cmd = dict(find='test', filter={}, hint=SON([('x', 1)]), comment='this is a test', maxScan=10, maxTimeMS=10000, max={'x': 10}, min={'x': -10}, returnKey=True, showRecordId=True, sort=SON([('_id', 1)]), projection={'x': False}, skip=1, batchSize=2, noCursorTimeout=True, allowPartialResults=True) self._test_find_options(query, cmd) def test_find_snapshot(self): # Test "snapshot" parameter separately, can't combine with "sort". query = dict(filter={}, snapshot=True) cmd = dict(find='test', filter={}, snapshot=True) self._test_find_options(query, cmd) def test_command_and_get_more(self): self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many( [{'x': 1} for _ in range(10)]) self.listener.results.clear() coll = self.client.pymongo_test.test # Test that we publish the unwrapped command. if self.client.is_mongos: coll = coll.with_options( read_preference=ReadPreference.PRIMARY_PREFERRED) cursor = coll.aggregate( [{'$project': {'_id': False, 'x': 1}}], batchSize=4) for _ in range(4): next(cursor) cursor_id = cursor.cursor_id results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand( SON([('aggregate', 'test'), ('pipeline', [{'$project': {'_id': False, 'x': 1}}]), ('cursor', {'batchSize': 4})]), started.command) self.assertEqual('aggregate', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('aggregate', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_cursor = {'id': cursor_id, 'ns': 'pymongo_test.test', 'firstBatch': [{'x': 1} for _ in range(4)]} self.assertEqualCommand(expected_cursor, succeeded.reply.get('cursor')) self.listener.results.clear() next(cursor) try: results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand( SON([('getMore', cursor_id), ('collection', 'test'), ('batchSize', 4)]), started.command) self.assertEqual('getMore', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('getMore', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_result = { 'cursor': {'id': cursor_id, 'ns': 'pymongo_test.test', 'nextBatch': [{'x': 1} for _ in range(4)]}, 'ok': 1.0} self.assertEqualReply(expected_result, succeeded.reply) finally: # Exhaust the cursor to avoid kill cursors. tuple(cursor) def test_get_more_failure(self): address = self.client.address coll = self.client.pymongo_test.test cursor_doc = {"id": 12345, "firstBatch": [], "ns": coll.full_name} cursor = CommandCursor(coll, cursor_doc, address) try: next(cursor) except Exception: pass results = self.listener.results started = results['started'][0] self.assertEqual(0, len(results['succeeded'])) failed = results['failed'][0] self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand( SON([('getMore', 12345), ('collection', 'test')]), started.command) self.assertEqual('getMore', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(failed, monitoring.CommandFailedEvent)) self.assertTrue(isinstance(failed.duration_micros, int)) self.assertEqual('getMore', failed.command_name) self.assertTrue(isinstance(failed.request_id, int)) self.assertEqual(cursor.address, failed.connection_id) self.assertEqual(0, failed.failure.get("ok")) @client_context.require_replica_set @client_context.require_secondaries_count(1) def test_not_master_error(self): address = next(iter(client_context.client.secondaries)) client = single_client(*address, event_listeners=[self.listener]) # Clear authentication command results from the listener. client.admin.command('ismaster') self.listener.results.clear() error = None try: client.pymongo_test.test.find_one_and_delete({}) except NotMasterError as exc: error = exc.errors results = self.listener.results started = results['started'][0] failed = results['failed'][0] self.assertEqual(0, len(results['succeeded'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertTrue( isinstance(failed, monitoring.CommandFailedEvent)) self.assertEqual('findAndModify', failed.command_name) self.assertEqual(address, failed.connection_id) self.assertEqual(0, failed.failure.get('ok')) self.assertTrue(isinstance(failed.request_id, int)) self.assertTrue(isinstance(failed.duration_micros, int)) self.assertEqual(error, failed.failure) @client_context.require_no_mongos def test_exhaust(self): self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many([{} for _ in range(10)]) self.listener.results.clear() cursor = self.client.pymongo_test.test.find( projection={'_id': False}, batch_size=5, cursor_type=CursorType.EXHAUST) next(cursor) cursor_id = cursor.cursor_id results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand( SON([('find', 'test'), ('filter', {}), ('projection', {'_id': False}), ('batchSize', 5)]), started.command) self.assertEqual('find', started.command_name) self.assertEqual(cursor.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('find', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_result = { 'cursor': {'id': cursor_id, 'ns': 'pymongo_test.test', 'firstBatch': [{} for _ in range(5)]}, 'ok': 1} self.assertEqualReply(expected_result, succeeded.reply) self.listener.results.clear() tuple(cursor) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand( SON([('getMore', cursor_id), ('collection', 'test'), ('batchSize', 5)]), started.command) self.assertEqual('getMore', started.command_name) self.assertEqual(cursor.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('getMore', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_result = { 'cursor': {'id': 0, 'ns': 'pymongo_test.test', 'nextBatch': [{} for _ in range(5)]}, 'ok': 1} self.assertEqualReply(expected_result, succeeded.reply) def test_kill_cursors(self): with client_knobs(kill_cursor_frequency=0.01): self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many([{} for _ in range(10)]) cursor = self.client.pymongo_test.test.find().batch_size(5) next(cursor) cursor_id = cursor.cursor_id self.listener.results.clear() cursor.close() time.sleep(2) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) # There could be more than one cursor_id here depending on # when the thread last ran. self.assertIn(cursor_id, started.command['cursors']) self.assertEqual('killCursors', started.command_name) self.assertIs(type(started.connection_id), tuple) self.assertEqual(cursor.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('killCursors', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertIs(type(succeeded.connection_id), tuple) self.assertEqual(cursor.address, succeeded.connection_id) # There could be more than one cursor_id here depending on # when the thread last ran. self.assertTrue(cursor_id in succeeded.reply['cursorsUnknown'] or cursor_id in succeeded.reply['cursorsKilled']) def test_non_bulk_writes(self): coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() # Implied write concern insert_one res = coll.insert_one({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': res.inserted_id, 'x': 1}])]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # Unacknowledged insert_one self.listener.results.clear() coll = coll.with_options(write_concern=WriteConcern(w=0)) res = coll.insert_one({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': res.inserted_id, 'x': 1}]), ('writeConcern', {'w': 0})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertEqualReply(succeeded.reply, {'ok': 1}) # Explicit write concern insert_one self.listener.results.clear() coll = coll.with_options(write_concern=WriteConcern(w=1)) res = coll.insert_one({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': res.inserted_id, 'x': 1}]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # delete_many self.listener.results.clear() res = coll.delete_many({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'x': 1}), ('limit', 0)])]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('delete', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(res.deleted_count, reply.get('n')) # replace_one self.listener.results.clear() oid = ObjectId() res = coll.replace_one({'_id': oid}, {'_id': oid, 'x': 1}, upsert=True) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'_id': oid}), ('u', {'_id': oid, 'x': 1}), ('multi', False), ('upsert', True)])]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) self.assertEqual([{'index': 0, '_id': oid}], reply.get('upserted')) # update_one self.listener.results.clear() res = coll.update_one({'x': 1}, {'$inc': {'x': 1}}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'x': 1}), ('u', {'$inc': {'x': 1}}), ('multi', False), ('upsert', False)])]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # update_many self.listener.results.clear() res = coll.update_many({'x': 2}, {'$inc': {'x': 1}}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'x': 2}), ('u', {'$inc': {'x': 1}}), ('multi', True), ('upsert', False)])]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # delete_one self.listener.results.clear() res = coll.delete_one({'x': 3}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'x': 3}), ('limit', 1)])]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('delete', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) self.assertEqual(0, coll.count()) # write errors coll.insert_one({'_id': 1}) try: self.listener.results.clear() coll.insert_one({'_id': 1}) except OperationFailure: pass results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': 1}]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(0, reply.get('n')) errors = reply.get('writeErrors') self.assertIsInstance(errors, list) error = errors[0] self.assertEqual(0, error.get('index')) self.assertIsInstance(error.get('code'), int) self.assertIsInstance(error.get('errmsg'), text_type) def test_legacy_writes(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() # Implied write concern insert _id = coll.insert({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': _id, 'x': 1}])]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # Unacknowledged insert self.listener.results.clear() _id = coll.insert({'x': 1}, w=0) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': _id, 'x': 1}]), ('writeConcern', {'w': 0})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertEqual(succeeded.reply, {'ok': 1}) # Explicit write concern insert self.listener.results.clear() _id = coll.insert({'x': 1}, w=1) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': _id, 'x': 1}]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # remove all self.listener.results.clear() res = coll.remove({'x': 1}, w=1) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'x': 1}), ('limit', 0)])]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('delete', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(res['n'], reply.get('n')) # upsert self.listener.results.clear() oid = ObjectId() coll.update({'_id': oid}, {'_id': oid, 'x': 1}, upsert=True, w=1) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'_id': oid}), ('u', {'_id': oid, 'x': 1}), ('multi', False), ('upsert', True)])]), ('writeConcern', {'w': 1})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) self.assertEqual([{'index': 0, '_id': oid}], reply.get('upserted')) # update one self.listener.results.clear() coll.update({'x': 1}, {'$inc': {'x': 1}}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'x': 1}), ('u', {'$inc': {'x': 1}}), ('multi', False), ('upsert', False)])])]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # update many self.listener.results.clear() coll.update({'x': 2}, {'$inc': {'x': 1}}, multi=True) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'x': 2}), ('u', {'$inc': {'x': 1}}), ('multi', True), ('upsert', False)])])]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # remove one self.listener.results.clear() coll.remove({'x': 3}, multi=False) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'x': 3}), ('limit', 1)])])]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('delete', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) self.assertEqual(0, coll.count()) def test_insert_many(self): # This always uses the bulk API. coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() big = 'x' * (1024 * 1024 * 4) docs = [{'_id': i, 'big': big} for i in range(6)] coll.insert_many(docs) results = self.listener.results started = results['started'] succeeded = results['succeeded'] self.assertEqual(0, len(results['failed'])) documents = [] count = 0 operation_id = started[0].operation_id self.assertIsInstance(operation_id, int) for start, succeed in zip(started, succeeded): self.assertIsInstance(start, monitoring.CommandStartedEvent) cmd = sanitize_cmd(start.command) self.assertEqual(['insert', 'ordered', 'documents'], list(cmd.keys())) self.assertEqual(coll.name, cmd['insert']) self.assertIs(True, cmd['ordered']) documents.extend(cmd['documents']) self.assertEqual('pymongo_test', start.database_name) self.assertEqual('insert', start.command_name) self.assertIsInstance(start.request_id, int) self.assertEqual(self.client.address, start.connection_id) self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) self.assertIsInstance(succeed.duration_micros, int) self.assertEqual(start.command_name, succeed.command_name) self.assertEqual(start.request_id, succeed.request_id) self.assertEqual(start.connection_id, succeed.connection_id) self.assertEqual(start.operation_id, operation_id) self.assertEqual(succeed.operation_id, operation_id) reply = succeed.reply self.assertEqual(1, reply.get('ok')) count += reply.get('n', 0) self.assertEqual(documents, docs) self.assertEqual(6, count) def test_legacy_insert_many(self): # On legacy servers this uses bulk OP_INSERT. with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() # Force two batches on legacy servers. big = 'x' * (1024 * 1024 * 12) docs = [{'_id': i, 'big': big} for i in range(6)] coll.insert(docs) results = self.listener.results started = results['started'] succeeded = results['succeeded'] self.assertEqual(0, len(results['failed'])) documents = [] count = 0 operation_id = started[0].operation_id self.assertIsInstance(operation_id, int) for start, succeed in zip(started, succeeded): self.assertIsInstance(start, monitoring.CommandStartedEvent) cmd = sanitize_cmd(start.command) self.assertEqual(['insert', 'ordered', 'documents'], list(cmd.keys())) self.assertEqual(coll.name, cmd['insert']) self.assertIs(True, cmd['ordered']) documents.extend(cmd['documents']) self.assertEqual('pymongo_test', start.database_name) self.assertEqual('insert', start.command_name) self.assertIsInstance(start.request_id, int) self.assertEqual(self.client.address, start.connection_id) self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) self.assertIsInstance(succeed.duration_micros, int) self.assertEqual(start.command_name, succeed.command_name) self.assertEqual(start.request_id, succeed.request_id) self.assertEqual(start.connection_id, succeed.connection_id) self.assertEqual(start.operation_id, operation_id) self.assertEqual(succeed.operation_id, operation_id) reply = succeed.reply self.assertEqual(1, reply.get('ok')) count += reply.get('n', 0) self.assertEqual(documents, docs) self.assertEqual(6, count) def test_bulk_write(self): coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() coll.bulk_write([InsertOne({'_id': 1}), UpdateOne({'_id': 1}, {'$set': {'x': 1}}), DeleteOne({'_id': 1})]) results = self.listener.results started = results['started'] succeeded = results['succeeded'] self.assertEqual(0, len(results['failed'])) operation_id = started[0].operation_id pairs = list(zip(started, succeeded)) self.assertEqual(3, len(pairs)) for start, succeed in pairs: self.assertIsInstance(start, monitoring.CommandStartedEvent) self.assertEqual('pymongo_test', start.database_name) self.assertIsInstance(start.request_id, int) self.assertEqual(self.client.address, start.connection_id) self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) self.assertIsInstance(succeed.duration_micros, int) self.assertEqual(start.command_name, succeed.command_name) self.assertEqual(start.request_id, succeed.request_id) self.assertEqual(start.connection_id, succeed.connection_id) self.assertEqual(start.operation_id, operation_id) self.assertEqual(succeed.operation_id, operation_id) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': 1}])]) self.assertEqualCommand(expected, started[0].command) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'_id': 1}), ('u', {'$set': {'x': 1}}), ('multi', False), ('upsert', False)])])]) self.assertEqualCommand(expected, started[1].command) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'_id': 1}), ('limit', 1)])])]) self.assertEqualCommand(expected, started[2].command) def test_write_errors(self): coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() try: coll.bulk_write([InsertOne({'_id': 1}), InsertOne({'_id': 1}), InsertOne({'_id': 1}), DeleteOne({'_id': 1})], ordered=False) except OperationFailure: pass results = self.listener.results started = results['started'] succeeded = results['succeeded'] self.assertEqual(0, len(results['failed'])) operation_id = started[0].operation_id pairs = list(zip(started, succeeded)) errors = [] for start, succeed in pairs: self.assertIsInstance(start, monitoring.CommandStartedEvent) self.assertEqual('pymongo_test', start.database_name) self.assertIsInstance(start.request_id, int) self.assertEqual(self.client.address, start.connection_id) self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) self.assertIsInstance(succeed.duration_micros, int) self.assertEqual(start.command_name, succeed.command_name) self.assertEqual(start.request_id, succeed.request_id) self.assertEqual(start.connection_id, succeed.connection_id) self.assertEqual(start.operation_id, operation_id) self.assertEqual(succeed.operation_id, operation_id) if 'writeErrors' in succeed.reply: errors.extend(succeed.reply['writeErrors']) self.assertEqual(2, len(errors)) fields = set(['index', 'code', 'errmsg']) for error in errors: self.assertTrue(fields.issubset(set(error))) def test_first_batch_helper(self): # Regardless of server version and use of helpers._first_batch # this test should still pass. self.listener.results.clear() tuple(self.client.pymongo_test.test.list_indexes()) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('listIndexes', 'test'), ('cursor', {})]) self.assertEqualCommand(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('listIndexes', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertTrue('cursor' in succeeded.reply) self.assertTrue('ok' in succeeded.reply) self.listener.results.clear() self.client.pymongo_test.current_op(True) started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('currentOp', 1), ('$all', True)]) self.assertEqualCommand(expected, started.command) self.assertEqual('admin', started.database_name) self.assertEqual('currentOp', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertTrue('inprog' in succeeded.reply) self.assertTrue('ok' in succeeded.reply) if not client_context.is_mongos: self.client.fsync(lock=True) self.listener.results.clear() self.client.unlock() # Wait for async unlock... wait_until( lambda: not self.client.is_locked, "unlock the database") started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = {'fsyncUnlock': 1} self.assertEqualCommand(expected, started.command) self.assertEqual('admin', started.database_name) self.assertEqual('fsyncUnlock', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertTrue('info' in succeeded.reply) self.assertTrue('ok' in succeeded.reply) def test_sensitive_commands(self): listeners = self.client._event_listeners self.listener.results.clear() cmd = SON([("getnonce", 1)]) listeners.publish_command_start( cmd, "pymongo_test", 12345, self.client.address) delta = datetime.timedelta(milliseconds=100) listeners.publish_command_success( delta, {'nonce': 'e474f4561c5eb40b', 'ok': 1.0}, "getnonce", 12345, self.client.address) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) self.assertEqual({}, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('getnonce', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertEqual(succeeded.duration_micros, 100000) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertEqual({}, succeeded.reply) class TestGlobalListener(PyMongoTestCase): @classmethod @client_context.require_connection def setUpClass(cls): cls.listener = EventListener() # We plan to call register(), which internally modifies _LISTENERS. cls.saved_listeners = copy.deepcopy(monitoring._LISTENERS) monitoring.register(cls.listener) cls.client = single_client() # Get one (authenticated) socket in the pool. cls.client.pymongo_test.command('ismaster') @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners def setUp(self): self.listener.results.clear() def test_simple(self): self.client.pymongo_test.command('ismaster') results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqualCommand(SON([('ismaster', 1)]), started.command) self.assertEqual('ismaster', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_binary.py0000644000076600000240000003336113245621354017322 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the Binary wrapper.""" import base64 import copy import pickle import sys import uuid sys.path[0:0] = [""] import bson from bson.binary import * from bson.codec_options import CodecOptions from bson.son import SON from test import client_context, unittest from pymongo.mongo_client import MongoClient class TestBinary(unittest.TestCase): @classmethod def setUpClass(cls): # Generated by the Java driver from_java = ( b'bAAAAAdfaWQAUCBQxkVm+XdxJ9tOBW5ld2d1aWQAEAAAAAMIQkfACFu' b'Z/0RustLOU/G6Am5ld2d1aWRzdHJpbmcAJQAAAGZmOTk1YjA4LWMwND' b'ctNDIwOC1iYWYxLTUzY2VkMmIyNmU0NAAAbAAAAAdfaWQAUCBQxkVm+' b'XdxJ9tPBW5ld2d1aWQAEAAAAANgS/xhRXXv8kfIec+dYdyCAm5ld2d1' b'aWRzdHJpbmcAJQAAAGYyZWY3NTQ1LTYxZmMtNGI2MC04MmRjLTYxOWR' b'jZjc5Yzg0NwAAbAAAAAdfaWQAUCBQxkVm+XdxJ9tQBW5ld2d1aWQAEA' b'AAAAPqREIbhZPUJOSdHCJIgaqNAm5ld2d1aWRzdHJpbmcAJQAAADI0Z' b'DQ5Mzg1LTFiNDItNDRlYS04ZGFhLTgxNDgyMjFjOWRlNAAAbAAAAAdf' b'aWQAUCBQxkVm+XdxJ9tRBW5ld2d1aWQAEAAAAANjQBn/aQuNfRyfNyx' b'29COkAm5ld2d1aWRzdHJpbmcAJQAAADdkOGQwYjY5LWZmMTktNDA2My' b'1hNDIzLWY0NzYyYzM3OWYxYwAAbAAAAAdfaWQAUCBQxkVm+XdxJ9tSB' b'W5ld2d1aWQAEAAAAAMtSv/Et1cAQUFHUYevqxaLAm5ld2d1aWRzdHJp' b'bmcAJQAAADQxMDA1N2I3LWM0ZmYtNGEyZC04YjE2LWFiYWY4NzUxNDc' b'0MQAA') cls.java_data = base64.b64decode(from_java) # Generated by the .net driver from_csharp = ( b'ZAAAABBfaWQAAAAAAAVuZXdndWlkABAAAAAD+MkoCd/Jy0iYJ7Vhl' b'iF3BAJuZXdndWlkc3RyaW5nACUAAAAwOTI4YzlmOC1jOWRmLTQ4Y2' b'ItOTgyNy1iNTYxOTYyMTc3MDQAAGQAAAAQX2lkAAEAAAAFbmV3Z3V' b'pZAAQAAAAA9MD0oXQe6VOp7mK4jkttWUCbmV3Z3VpZHN0cmluZwAl' b'AAAAODVkMjAzZDMtN2JkMC00ZWE1LWE3YjktOGFlMjM5MmRiNTY1A' b'ABkAAAAEF9pZAACAAAABW5ld2d1aWQAEAAAAAPRmIO2auc/Tprq1Z' b'oQ1oNYAm5ld2d1aWRzdHJpbmcAJQAAAGI2ODM5OGQxLWU3NmEtNGU' b'zZi05YWVhLWQ1OWExMGQ2ODM1OAAAZAAAABBfaWQAAwAAAAVuZXdn' b'dWlkABAAAAADISpriopuTEaXIa7arYOCFAJuZXdndWlkc3RyaW5nA' b'CUAAAA4YTZiMmEyMS02ZThhLTQ2NGMtOTcyMS1hZWRhYWQ4MzgyMT' b'QAAGQAAAAQX2lkAAQAAAAFbmV3Z3VpZAAQAAAAA98eg0CFpGlPihP' b'MwOmYGOMCbmV3Z3VpZHN0cmluZwAlAAAANDA4MzFlZGYtYTQ4NS00' b'ZjY5LThhMTMtY2NjMGU5OTgxOGUzAAA=') cls.csharp_data = base64.b64decode(from_csharp) def test_binary(self): a_string = "hello world" a_binary = Binary(b"hello world") self.assertTrue(a_binary.startswith(b"hello")) self.assertTrue(a_binary.endswith(b"world")) self.assertTrue(isinstance(a_binary, Binary)) self.assertFalse(isinstance(a_string, Binary)) def test_exceptions(self): self.assertRaises(TypeError, Binary, None) self.assertRaises(TypeError, Binary, u"hello") self.assertRaises(TypeError, Binary, 5) self.assertRaises(TypeError, Binary, 10.2) self.assertRaises(TypeError, Binary, b"hello", None) self.assertRaises(TypeError, Binary, b"hello", "100") self.assertRaises(ValueError, Binary, b"hello", -1) self.assertRaises(ValueError, Binary, b"hello", 256) self.assertTrue(Binary(b"hello", 0)) self.assertTrue(Binary(b"hello", 255)) def test_subtype(self): one = Binary(b"hello") self.assertEqual(one.subtype, 0) two = Binary(b"hello", 2) self.assertEqual(two.subtype, 2) three = Binary(b"hello", 100) self.assertEqual(three.subtype, 100) def test_equality(self): two = Binary(b"hello") three = Binary(b"hello", 100) self.assertNotEqual(two, three) self.assertEqual(three, Binary(b"hello", 100)) self.assertEqual(two, Binary(b"hello")) self.assertNotEqual(two, Binary(b"hello ")) self.assertNotEqual(b"hello", Binary(b"hello")) # Explicitly test inequality self.assertFalse(three != Binary(b"hello", 100)) self.assertFalse(two != Binary(b"hello")) def test_repr(self): one = Binary(b"hello world") self.assertEqual(repr(one), "Binary(%s, 0)" % (repr(b"hello world"),)) two = Binary(b"hello world", 2) self.assertEqual(repr(two), "Binary(%s, 2)" % (repr(b"hello world"),)) three = Binary(b"\x08\xFF") self.assertEqual(repr(three), "Binary(%s, 0)" % (repr(b"\x08\xFF"),)) four = Binary(b"\x08\xFF", 2) self.assertEqual(repr(four), "Binary(%s, 2)" % (repr(b"\x08\xFF"),)) five = Binary(b"test", 100) self.assertEqual(repr(five), "Binary(%s, 100)" % (repr(b"test"),)) def test_hash(self): one = Binary(b"hello world") two = Binary(b"hello world", 42) self.assertEqual(hash(Binary(b"hello world")), hash(one)) self.assertNotEqual(hash(one), hash(two)) self.assertEqual(hash(Binary(b"hello world", 42)), hash(two)) def test_uuid_subtype_4(self): """uuid_representation should be ignored when decoding subtype 4.""" expected_uuid = uuid.uuid4() doc = {"uuid": Binary(expected_uuid.bytes, 4)} encoded = bson.BSON.encode(doc) for uuid_representation in ALL_UUID_REPRESENTATIONS: options = CodecOptions(uuid_representation=uuid_representation) self.assertEqual(expected_uuid, encoded.decode(options)["uuid"]) def test_legacy_java_uuid(self): # Test decoding data = self.java_data docs = bson.decode_all(data, CodecOptions(SON, False, PYTHON_LEGACY)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, STANDARD)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, CSHARP_LEGACY)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, JAVA_LEGACY)) for d in docs: self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring'])) # Test encoding encoded = b''.join([ bson.BSON.encode(doc, False, CodecOptions(uuid_representation=PYTHON_LEGACY)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=STANDARD)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=CSHARP_LEGACY)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=JAVA_LEGACY)) for doc in docs]) self.assertEqual(data, encoded) @client_context.require_connection def test_legacy_java_uuid_roundtrip(self): data = self.java_data docs = bson.decode_all(data, CodecOptions(SON, False, JAVA_LEGACY)) client_context.client.pymongo_test.drop_collection('java_uuid') db = client_context.client.pymongo_test coll = db.get_collection( 'java_uuid', CodecOptions(uuid_representation=JAVA_LEGACY)) coll.insert_many(docs) self.assertEqual(5, coll.count()) for d in coll.find(): self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring'])) coll = db.get_collection( 'java_uuid', CodecOptions(uuid_representation=PYTHON_LEGACY)) for d in coll.find(): self.assertNotEqual(d['newguid'], d['newguidstring']) client_context.client.pymongo_test.drop_collection('java_uuid') def test_legacy_csharp_uuid(self): data = self.csharp_data # Test decoding docs = bson.decode_all(data, CodecOptions(SON, False, PYTHON_LEGACY)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, STANDARD)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, JAVA_LEGACY)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, CSHARP_LEGACY)) for d in docs: self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring'])) # Test encoding encoded = b''.join([ bson.BSON.encode(doc, False, CodecOptions(uuid_representation=PYTHON_LEGACY)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join([ bson.BSON.encode(doc, False, CodecOptions(uuid_representation=STANDARD)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=JAVA_LEGACY)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=CSHARP_LEGACY)) for doc in docs]) self.assertEqual(data, encoded) @client_context.require_connection def test_legacy_csharp_uuid_roundtrip(self): data = self.csharp_data docs = bson.decode_all(data, CodecOptions(SON, False, CSHARP_LEGACY)) client_context.client.pymongo_test.drop_collection('csharp_uuid') db = client_context.client.pymongo_test coll = db.get_collection( 'csharp_uuid', CodecOptions(uuid_representation=CSHARP_LEGACY)) coll.insert_many(docs) self.assertEqual(5, coll.count()) for d in coll.find(): self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring'])) coll = db.get_collection( 'csharp_uuid', CodecOptions(uuid_representation=PYTHON_LEGACY)) for d in coll.find(): self.assertNotEqual(d['newguid'], d['newguidstring']) client_context.client.pymongo_test.drop_collection('csharp_uuid') def test_uri_to_uuid(self): uri = "mongodb://foo/?uuidrepresentation=csharpLegacy" client = MongoClient(uri, connect=False) self.assertEqual( client.pymongo_test.test.codec_options.uuid_representation, CSHARP_LEGACY) @client_context.require_connection def test_uuid_queries(self): db = client_context.client.pymongo_test coll = db.test coll.drop() uu = uuid.uuid4() coll.insert_one({'uuid': Binary(uu.bytes, 3)}) self.assertEqual(1, coll.count()) # Test UUIDLegacy queries. coll = db.get_collection("test", CodecOptions(uuid_representation=STANDARD)) self.assertEqual(0, coll.find({'uuid': uu}).count()) cur = coll.find({'uuid': UUIDLegacy(uu)}) self.assertEqual(1, cur.count()) retrieved = next(cur) self.assertEqual(uu, retrieved['uuid']) # Test regular UUID queries (using subtype 4). coll.insert_one({'uuid': uu}) self.assertEqual(2, coll.count()) cur = coll.find({'uuid': uu}) self.assertEqual(1, cur.count()) retrieved = next(cur) self.assertEqual(uu, retrieved['uuid']) # Test both. cur = coll.find({'uuid': {'$in': [uu, UUIDLegacy(uu)]}}) self.assertEqual(2, cur.count()) coll.drop() def test_pickle(self): b1 = Binary(b'123', 2) # For testing backwards compatibility with pre-2.4 pymongo if PY3: p = (b"\x80\x03cbson.binary\nBinary\nq\x00C\x03123q\x01\x85q" b"\x02\x81q\x03}q\x04X\x10\x00\x00\x00_Binary__subtypeq" b"\x05K\x02sb.") else: p = (b"ccopy_reg\n_reconstructor\np0\n(cbson.binary\nBinary\np1\nc" b"__builtin__\nstr\np2\nS'123'\np3\ntp4\nRp5\n(dp6\nS'_Binary" b"__subtype'\np7\nI2\nsb.") if not sys.version.startswith('3.0'): self.assertEqual(b1, pickle.loads(p)) for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.assertEqual(b1, pickle.loads(pickle.dumps(b1, proto))) uu = uuid.uuid4() uul = UUIDLegacy(uu) self.assertEqual(uul, copy.copy(uul)) self.assertEqual(uul, copy.deepcopy(uul)) for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.assertEqual(uul, pickle.loads(pickle.dumps(uul, proto))) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_server.py0000644000076600000240000000215013245617773017346 0ustar shanestaff00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the server module.""" import sys sys.path[0:0] = [""] from pymongo.ismaster import IsMaster from pymongo.server import Server from pymongo.server_description import ServerDescription from test import unittest class TestServer(unittest.TestCase): def test_repr(self): ismaster = IsMaster({'ok': 1}) sd = ServerDescription(('localhost', 27017), ismaster) server = Server(sd, pool=object(), monitor=object()) self.assertTrue('Standalone' in str(server)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_ssl.py0000644000076600000240000006211013245621354016631 0ustar shanestaff00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for SSL support.""" import os import socket import sys sys.path[0:0] = [""] try: from urllib.parse import quote_plus except ImportError: # Python 2 from urllib import quote_plus from pymongo import MongoClient, ssl_support from pymongo.errors import (ConfigurationError, ConnectionFailure, OperationFailure) from pymongo.ssl_support import HAVE_SSL, get_ssl_context, validate_cert_reqs from pymongo.write_concern import WriteConcern from test import (IntegrationTest, client_context, db_pwd, db_user, SkipTest, unittest, HAVE_IPADDRESS) from test.utils import remove_all_users, connected if HAVE_SSL: import ssl CERT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'certificates') CLIENT_PEM = os.path.join(CERT_PATH, 'client.pem') CLIENT_ENCRYPTED_PEM = os.path.join(CERT_PATH, 'client_encrypted.pem') CA_PEM = os.path.join(CERT_PATH, 'ca.pem') CRL_PEM = os.path.join(CERT_PATH, 'crl.pem') MONGODB_X509_USERNAME = ( "C=US,ST=California,L=Palo Alto,O=,OU=Drivers,CN=client") # To fully test this start a mongod instance (built with SSL support) like so: # mongod --dbpath /path/to/data/directory --sslOnNormalPorts \ # --sslPEMKeyFile /path/to/pymongo/test/certificates/server.pem \ # --sslCAFile /path/to/pymongo/test/certificates/ca.pem \ # --sslWeakCertificateValidation # Also, make sure you have 'server' as an alias for localhost in /etc/hosts # # Note: For all replica set tests to pass, the replica set configuration must # use 'localhost' for the hostname of all hosts. class TestClientSSL(unittest.TestCase): @unittest.skipIf(HAVE_SSL, "The ssl module is available, can't test what " "happens without it.") def test_no_ssl_module(self): # Explicit self.assertRaises(ConfigurationError, MongoClient, ssl=True) # Implied self.assertRaises(ConfigurationError, MongoClient, ssl_certfile=CLIENT_PEM) @unittest.skipUnless(HAVE_SSL, "The ssl module is not available.") def test_config_ssl(self): # Tests various ssl configurations self.assertRaises(ValueError, MongoClient, ssl='foo') self.assertRaises(ConfigurationError, MongoClient, ssl=False, ssl_certfile=CLIENT_PEM) self.assertRaises(TypeError, MongoClient, ssl=0) self.assertRaises(TypeError, MongoClient, ssl=5.5) self.assertRaises(TypeError, MongoClient, ssl=[]) self.assertRaises(IOError, MongoClient, ssl_certfile="NoSuchFile") self.assertRaises(TypeError, MongoClient, ssl_certfile=True) self.assertRaises(TypeError, MongoClient, ssl_certfile=[]) self.assertRaises(IOError, MongoClient, ssl_keyfile="NoSuchFile") self.assertRaises(TypeError, MongoClient, ssl_keyfile=True) self.assertRaises(TypeError, MongoClient, ssl_keyfile=[]) # Test invalid combinations self.assertRaises(ConfigurationError, MongoClient, ssl=False, ssl_keyfile=CLIENT_PEM) self.assertRaises(ConfigurationError, MongoClient, ssl=False, ssl_certfile=CLIENT_PEM) self.assertRaises(ConfigurationError, MongoClient, ssl=False, ssl_keyfile=CLIENT_PEM, ssl_certfile=CLIENT_PEM) self.assertRaises( ValueError, validate_cert_reqs, 'ssl_cert_reqs', 3) self.assertRaises( ValueError, validate_cert_reqs, 'ssl_cert_reqs', -1) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', None), None) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', ssl.CERT_NONE), ssl.CERT_NONE) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', ssl.CERT_OPTIONAL), ssl.CERT_OPTIONAL) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', ssl.CERT_REQUIRED), ssl.CERT_REQUIRED) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 0), ssl.CERT_NONE) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 1), ssl.CERT_OPTIONAL) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 2), ssl.CERT_REQUIRED) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 'CERT_NONE'), ssl.CERT_NONE) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 'CERT_OPTIONAL'), ssl.CERT_OPTIONAL) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 'CERT_REQUIRED'), ssl.CERT_REQUIRED) class TestSSL(IntegrationTest): def assertClientWorks(self, client): coll = client.pymongo_test.ssl_test.with_options( write_concern=WriteConcern(w=client_context.w)) coll.drop() coll.insert_one({'ssl': True}) self.assertTrue(coll.find_one()['ssl']) coll.drop() @classmethod @unittest.skipUnless(HAVE_SSL, "The ssl module is not available.") def setUpClass(cls): super(TestSSL, cls).setUpClass() # MongoClient should connect to the primary by default. cls.saved_port = MongoClient.PORT MongoClient.PORT = client_context.port @classmethod def tearDownClass(cls): MongoClient.PORT = cls.saved_port super(TestSSL, cls).tearDownClass() @client_context.require_ssl def test_simple_ssl(self): # Expects the server to be running with ssl and with # no --sslPEMKeyFile or with --sslWeakCertificateValidation self.assertClientWorks(self.client) @client_context.require_ssl_certfile def test_ssl_pem_passphrase(self): # Expects the server to be running with server.pem and ca.pem # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem vi = sys.version_info if vi[0] == 2 and vi < (2, 7, 9) or vi[0] == 3 and vi < (3, 3): self.assertRaises( ConfigurationError, MongoClient, 'localhost', ssl=True, ssl_certfile=CLIENT_ENCRYPTED_PEM, ssl_pem_passphrase="clientpassword", ssl_ca_certs=CA_PEM, serverSelectionTimeoutMS=100) else: connected(MongoClient('localhost', ssl=True, ssl_certfile=CLIENT_ENCRYPTED_PEM, ssl_pem_passphrase="clientpassword", ssl_ca_certs=CA_PEM, serverSelectionTimeoutMS=100, **self.credentials)) uri_fmt = ("mongodb://localhost/?ssl=true" "&ssl_certfile=%s&ssl_pem_passphrase=clientpassword" "&ssl_ca_certs=%s&serverSelectionTimeoutMS=100") connected(MongoClient(uri_fmt % (CLIENT_ENCRYPTED_PEM, CA_PEM), **self.credentials)) @client_context.require_ssl_certfile @client_context.require_no_auth def test_cert_ssl_implicitly_set(self): # Expects the server to be running with server.pem and ca.pem # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # # test that setting ssl_certfile causes ssl to be set to True client = MongoClient(client_context.host, client_context.port, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) response = client.admin.command('ismaster') if 'setName' in response: client = MongoClient(client_context.pair, replicaSet=response['setName'], w=len(response['hosts']), ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) self.assertClientWorks(client) @client_context.require_ssl_certfile @client_context.require_no_auth def test_cert_ssl_validation(self): # Expects the server to be running with server.pem and ca.pem # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # client = MongoClient('localhost', ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM) response = client.admin.command('ismaster') if 'setName' in response: if response['primary'].split(":")[0] != 'localhost': raise SkipTest("No hosts in the replicaset for 'localhost'. " "Cannot validate hostname in the certificate") client = MongoClient('localhost', replicaSet=response['setName'], w=len(response['hosts']), ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM) self.assertClientWorks(client) # Python 2.6 often can't read SANs from the peer cert. # http://bugs.python.org/issue13034 if HAVE_IPADDRESS and sys.version_info[:2] > (2, 6): client = MongoClient('127.0.0.1', ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM) self.assertClientWorks(client) @client_context.require_ssl_certfile @client_context.require_no_auth def test_cert_ssl_uri_support(self): # Expects the server to be running with server.pem and ca.pem # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # uri_fmt = ("mongodb://localhost/?ssl=true&ssl_certfile=%s&ssl_cert_reqs" "=%s&ssl_ca_certs=%s&ssl_match_hostname=true") client = MongoClient(uri_fmt % (CLIENT_PEM, 'CERT_REQUIRED', CA_PEM)) self.assertClientWorks(client) @client_context.require_ssl_certfile @client_context.require_no_auth def test_cert_ssl_validation_optional(self): # Expects the server to be running with server.pem and ca.pem # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # client = MongoClient('localhost', ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_OPTIONAL, ssl_ca_certs=CA_PEM) response = client.admin.command('ismaster') if 'setName' in response: if response['primary'].split(":")[0] != 'localhost': raise SkipTest("No hosts in the replicaset for 'localhost'. " "Cannot validate hostname in the certificate") client = MongoClient('localhost', replicaSet=response['setName'], w=len(response['hosts']), ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_OPTIONAL, ssl_ca_certs=CA_PEM) self.assertClientWorks(client) @client_context.require_ssl_certfile @client_context.require_server_resolvable def test_cert_ssl_validation_hostname_matching(self): # Expects the server to be running with server.pem and ca.pem # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # response = self.client.admin.command('ismaster') with self.assertRaises(ConnectionFailure): connected(MongoClient('server', ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, serverSelectionTimeoutMS=500, **self.credentials)) connected(MongoClient('server', ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, ssl_match_hostname=False, serverSelectionTimeoutMS=500, **self.credentials)) if 'setName' in response: with self.assertRaises(ConnectionFailure): connected(MongoClient('server', replicaSet=response['setName'], ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, serverSelectionTimeoutMS=500, **self.credentials)) connected(MongoClient('server', replicaSet=response['setName'], ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, ssl_match_hostname=False, serverSelectionTimeoutMS=500, **self.credentials)) @client_context.require_ssl_certfile def test_ssl_crlfile_support(self): if not hasattr(ssl, 'VERIFY_CRL_CHECK_LEAF'): self.assertRaises( ConfigurationError, MongoClient, 'localhost', ssl=True, ssl_ca_certs=CA_PEM, ssl_crlfile=CRL_PEM, serverSelectionTimeoutMS=100) else: connected(MongoClient('localhost', ssl=True, ssl_ca_certs=CA_PEM, serverSelectionTimeoutMS=100, **self.credentials)) with self.assertRaises(ConnectionFailure): connected(MongoClient('localhost', ssl=True, ssl_ca_certs=CA_PEM, ssl_crlfile=CRL_PEM, serverSelectionTimeoutMS=100, **self.credentials)) uri_fmt = ("mongodb://localhost/?ssl=true&" "ssl_ca_certs=%s&serverSelectionTimeoutMS=100") connected(MongoClient(uri_fmt % (CA_PEM,), **self.credentials)) uri_fmt = ("mongodb://localhost/?ssl=true&ssl_crlfile=%s" "&ssl_ca_certs=%s&serverSelectionTimeoutMS=100") with self.assertRaises(ConnectionFailure): connected(MongoClient(uri_fmt % (CRL_PEM, CA_PEM), **self.credentials)) @client_context.require_ssl_certfile @client_context.require_server_resolvable def test_validation_with_system_ca_certs(self): # Expects the server to be running with server.pem and ca.pem. # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslWeakCertificateValidation # if sys.platform == "win32": raise SkipTest("Can't test system ca certs on Windows.") if sys.version_info < (2, 7, 9): raise SkipTest("Can't load system CA certificates.") # Tell OpenSSL where CA certificates live. os.environ['SSL_CERT_FILE'] = CA_PEM try: with self.assertRaises(ConnectionFailure): # Server cert is verified but hostname matching fails connected(MongoClient('server', ssl=True, serverSelectionTimeoutMS=100, **self.credentials)) # Server cert is verified. Disable hostname matching. connected(MongoClient('server', ssl=True, ssl_match_hostname=False, serverSelectionTimeoutMS=100, **self.credentials)) # Server cert and hostname are verified. connected(MongoClient('localhost', ssl=True, serverSelectionTimeoutMS=100, **self.credentials)) # Server cert and hostname are verified. connected( MongoClient( 'mongodb://localhost/?ssl=true&serverSelectionTimeoutMS=100', **self.credentials)) finally: os.environ.pop('SSL_CERT_FILE') def test_system_certs_config_error(self): ctx = get_ssl_context(None, None, None, None, ssl.CERT_NONE, None) if ((sys.platform != "win32" and hasattr(ctx, "set_default_verify_paths")) or hasattr(ctx, "load_default_certs")): raise SkipTest( "Can't test when system CA certificates are loadable.") have_certifi = ssl_support.HAVE_CERTIFI have_wincertstore = ssl_support.HAVE_WINCERTSTORE # Force the test regardless of environment. ssl_support.HAVE_CERTIFI = False ssl_support.HAVE_WINCERTSTORE = False try: with self.assertRaises(ConfigurationError): MongoClient("mongodb://localhost/?ssl=true") finally: ssl_support.HAVE_CERTIFI = have_certifi ssl_support.HAVE_WINCERTSTORE = have_wincertstore def test_certifi_support(self): if hasattr(ssl, "SSLContext"): # SSLSocket doesn't provide ca_certs attribute on pythons # with SSLContext and SSLContext provides no information # about ca_certs. raise SkipTest("Can't test when SSLContext available.") if not ssl_support.HAVE_CERTIFI: raise SkipTest("Need certifi to test certifi support.") have_wincertstore = ssl_support.HAVE_WINCERTSTORE # Force the test on Windows, regardless of environment. ssl_support.HAVE_WINCERTSTORE = False try: ctx = get_ssl_context(None, None, None, CA_PEM, ssl.CERT_REQUIRED, None) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, CA_PEM) ctx = get_ssl_context(None, None, None, None, None, None) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, ssl_support.certifi.where()) finally: ssl_support.HAVE_WINCERTSTORE = have_wincertstore def test_wincertstore(self): if sys.platform != "win32": raise SkipTest("Only valid on Windows.") if hasattr(ssl, "SSLContext"): # SSLSocket doesn't provide ca_certs attribute on pythons # with SSLContext and SSLContext provides no information # about ca_certs. raise SkipTest("Can't test when SSLContext available.") if not ssl_support.HAVE_WINCERTSTORE: raise SkipTest("Need wincertstore to test wincertstore.") ctx = get_ssl_context(None, None, None, CA_PEM, ssl.CERT_REQUIRED, None) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, CA_PEM) ctx = get_ssl_context(None, None, None, None, None, None) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, ssl_support._WINCERTS.name) @client_context.require_auth @client_context.require_ssl_certfile def test_mongodb_x509_auth(self): host, port = client_context.host, client_context.port ssl_client = MongoClient( client_context.pair, ssl=True, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) self.addCleanup(remove_all_users, ssl_client['$external']) ssl_client.admin.authenticate(db_user, db_pwd) # Give x509 user all necessary privileges. client_context.create_user('$external', MONGODB_X509_USERNAME, roles=[ {'role': 'readWriteAnyDatabase', 'db': 'admin'}, {'role': 'userAdminAnyDatabase', 'db': 'admin'}]) noauth = MongoClient( client_context.pair, ssl=True, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) self.assertRaises(OperationFailure, noauth.pymongo_test.test.count) auth = MongoClient( client_context.pair, authMechanism='MONGODB-X509', ssl=True, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) if client_context.version.at_least(3, 3, 12): # No error auth.pymongo_test.test.find_one() else: # Should require a username with self.assertRaises(ConfigurationError): auth.pymongo_test.test.find_one() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), host, port)) client = MongoClient(uri, ssl=True, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) # No error client.pymongo_test.test.find_one() uri = 'mongodb://%s:%d/?authMechanism=MONGODB-X509' % (host, port) client = MongoClient(uri, ssl=True, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) if client_context.version.at_least(3, 3, 12): # No error client.pymongo_test.test.find_one() else: # Should require a username with self.assertRaises(ConfigurationError): client.pymongo_test.test.find_one() # Auth should fail if username and certificate do not match uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus("not the username"), host, port)) bad_client = MongoClient( uri, ssl=True, ssl_cert_reqs="CERT_NONE", ssl_certfile=CLIENT_PEM) with self.assertRaises(OperationFailure): bad_client.pymongo_test.test.find_one() bad_client = MongoClient( client_context.pair, username="not the username", authMechanism='MONGODB-X509', ssl=True, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) with self.assertRaises(OperationFailure): bad_client.pymongo_test.test.find_one() # Invalid certificate (using CA certificate as client certificate) uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), host, port)) try: connected(MongoClient(uri, ssl=True, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CA_PEM, serverSelectionTimeoutMS=100)) except (ConnectionFailure, ConfigurationError): pass else: self.fail("Invalid certificate accepted.") if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_server_selection.py0000644000076600000240000000207113245621354021403 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the topology module's Server Selection Spec implementation.""" import os import sys sys.path[0:0] = [""] from test import unittest from test.utils_selection_tests import create_selection_tests # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), os.path.join('server_selection', 'server_selection')) class TestAllScenarios(create_selection_tests(_TEST_PATH)): pass if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_grid_file.py0000644000076600000240000005011213245621354017753 0ustar shanestaff00000000000000# -*- coding: utf-8 -*- # # Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the grid_file module. """ import datetime import sys sys.path[0:0] = [""] from bson.objectid import ObjectId from bson.py3compat import StringIO from gridfs import GridFS from gridfs.grid_file import (DEFAULT_CHUNK_SIZE, _SEEK_CUR, _SEEK_END, GridIn, GridOut, GridOutCursor) from gridfs.errors import NoFile from pymongo import MongoClient from pymongo.errors import ConfigurationError, ServerSelectionTimeoutError from test import (IntegrationTest, unittest, qcheck) from test.utils import rs_or_single_client class TestGridFileNoConnect(unittest.TestCase): """Test GridFile features on a client that does not connect. """ @classmethod def setUpClass(cls): cls.db = MongoClient(connect=False).pymongo_test def test_grid_in_custom_opts(self): self.assertRaises(TypeError, GridIn, "foo") a = GridIn(self.db.fs, _id=5, filename="my_file", contentType="text/html", chunkSize=1000, aliases=["foo"], metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") self.assertEqual(5, a._id) self.assertEqual("my_file", a.filename) self.assertEqual("my_file", a.name) self.assertEqual("text/html", a.content_type) self.assertEqual(1000, a.chunk_size) self.assertEqual(["foo"], a.aliases) self.assertEqual({"foo": 1, "bar": 2}, a.metadata) self.assertEqual(3, a.bar) self.assertEqual("hello", a.baz) self.assertRaises(AttributeError, getattr, a, "mike") b = GridIn(self.db.fs, content_type="text/html", chunk_size=1000, baz=100) self.assertEqual("text/html", b.content_type) self.assertEqual(1000, b.chunk_size) self.assertEqual(100, b.baz) class TestGridFile(IntegrationTest): def setUp(self): self.db.drop_collection('fs.files') self.db.drop_collection('fs.chunks') def test_basic(self): f = GridIn(self.db.fs, filename="test") f.write(b"hello world") f.close() self.assertEqual(1, self.db.fs.files.find().count()) self.assertEqual(1, self.db.fs.chunks.find().count()) g = GridOut(self.db.fs, f._id) self.assertEqual(b"hello world", g.read()) # make sure it's still there... g = GridOut(self.db.fs, f._id) self.assertEqual(b"hello world", g.read()) f = GridIn(self.db.fs, filename="test") f.close() self.assertEqual(2, self.db.fs.files.find().count()) self.assertEqual(1, self.db.fs.chunks.find().count()) g = GridOut(self.db.fs, f._id) self.assertEqual(b"", g.read()) # test that reading 0 returns proper type self.assertEqual(b"", g.read(0)) def test_md5(self): f = GridIn(self.db.fs) f.write(b"hello world\n") f.close() self.assertEqual("6f5902ac237024bdd0c176cb93063dc4", f.md5) def test_alternate_collection(self): self.db.alt.files.delete_many({}) self.db.alt.chunks.delete_many({}) f = GridIn(self.db.alt) f.write(b"hello world") f.close() self.assertEqual(1, self.db.alt.files.find().count()) self.assertEqual(1, self.db.alt.chunks.find().count()) g = GridOut(self.db.alt, f._id) self.assertEqual(b"hello world", g.read()) # test that md5 still works... self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5) def test_grid_in_default_opts(self): self.assertRaises(TypeError, GridIn, "foo") a = GridIn(self.db.fs) self.assertTrue(isinstance(a._id, ObjectId)) self.assertRaises(AttributeError, setattr, a, "_id", 5) self.assertEqual(None, a.filename) self.assertEqual(None, a.name) a.filename = "my_file" self.assertEqual("my_file", a.filename) self.assertEqual("my_file", a.name) self.assertEqual(None, a.content_type) a.content_type = "text/html" self.assertEqual("text/html", a.content_type) self.assertRaises(AttributeError, getattr, a, "length") self.assertRaises(AttributeError, setattr, a, "length", 5) self.assertEqual(255 * 1024, a.chunk_size) self.assertRaises(AttributeError, setattr, a, "chunk_size", 5) self.assertRaises(AttributeError, getattr, a, "upload_date") self.assertRaises(AttributeError, setattr, a, "upload_date", 5) self.assertRaises(AttributeError, getattr, a, "aliases") a.aliases = ["foo"] self.assertEqual(["foo"], a.aliases) self.assertRaises(AttributeError, getattr, a, "metadata") a.metadata = {"foo": 1} self.assertEqual({"foo": 1}, a.metadata) self.assertRaises(AttributeError, setattr, a, "md5", 5) a.close() a.forty_two = 42 self.assertEqual(42, a.forty_two) self.assertTrue(isinstance(a._id, ObjectId)) self.assertRaises(AttributeError, setattr, a, "_id", 5) self.assertEqual("my_file", a.filename) self.assertEqual("my_file", a.name) self.assertEqual("text/html", a.content_type) self.assertEqual(0, a.length) self.assertRaises(AttributeError, setattr, a, "length", 5) self.assertEqual(255 * 1024, a.chunk_size) self.assertRaises(AttributeError, setattr, a, "chunk_size", 5) self.assertTrue(isinstance(a.upload_date, datetime.datetime)) self.assertRaises(AttributeError, setattr, a, "upload_date", 5) self.assertEqual(["foo"], a.aliases) self.assertEqual({"foo": 1}, a.metadata) self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", a.md5) self.assertRaises(AttributeError, setattr, a, "md5", 5) # Make sure custom attributes that were set both before and after # a.close() are reflected in b. PYTHON-411. b = GridFS(self.db).get_last_version(filename=a.filename) self.assertEqual(a.metadata, b.metadata) self.assertEqual(a.aliases, b.aliases) self.assertEqual(a.forty_two, b.forty_two) def test_grid_out_default_opts(self): self.assertRaises(TypeError, GridOut, "foo") gout = GridOut(self.db.fs, 5) with self.assertRaises(NoFile): gout.name a = GridIn(self.db.fs) a.close() b = GridOut(self.db.fs, a._id) self.assertEqual(a._id, b._id) self.assertEqual(0, b.length) self.assertEqual(None, b.content_type) self.assertEqual(None, b.name) self.assertEqual(None, b.filename) self.assertEqual(255 * 1024, b.chunk_size) self.assertTrue(isinstance(b.upload_date, datetime.datetime)) self.assertEqual(None, b.aliases) self.assertEqual(None, b.metadata) self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", b.md5) for attr in ["_id", "name", "content_type", "length", "chunk_size", "upload_date", "aliases", "metadata", "md5"]: self.assertRaises(AttributeError, setattr, b, attr, 5) def test_grid_out_cursor_options(self): self.assertRaises(TypeError, GridOutCursor.__init__, self.db.fs, {}, projection={"filename": 1}) cursor = GridOutCursor(self.db.fs, {}) cursor_clone = cursor.clone() cursor_dict = cursor.__dict__.copy() cursor_dict.pop('_Cursor__session') cursor_clone_dict = cursor_clone.__dict__.copy() cursor_clone_dict.pop('_Cursor__session') self.assertEqual(cursor_dict, cursor_clone_dict) self.assertRaises(NotImplementedError, cursor.add_option, 0) self.assertRaises(NotImplementedError, cursor.remove_option, 0) def test_grid_out_custom_opts(self): one = GridIn(self.db.fs, _id=5, filename="my_file", contentType="text/html", chunkSize=1000, aliases=["foo"], metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") one.write(b"hello world") one.close() two = GridOut(self.db.fs, 5) self.assertEqual("my_file", two.name) self.assertEqual("my_file", two.filename) self.assertEqual(5, two._id) self.assertEqual(11, two.length) self.assertEqual("text/html", two.content_type) self.assertEqual(1000, two.chunk_size) self.assertTrue(isinstance(two.upload_date, datetime.datetime)) self.assertEqual(["foo"], two.aliases) self.assertEqual({"foo": 1, "bar": 2}, two.metadata) self.assertEqual(3, two.bar) self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", two.md5) for attr in ["_id", "name", "content_type", "length", "chunk_size", "upload_date", "aliases", "metadata", "md5"]: self.assertRaises(AttributeError, setattr, two, attr, 5) def test_grid_out_file_document(self): one = GridIn(self.db.fs) one.write(b"foo bar") one.close() two = GridOut(self.db.fs, file_document=self.db.fs.files.find_one()) self.assertEqual(b"foo bar", two.read()) three = GridOut(self.db.fs, 5, file_document=self.db.fs.files.find_one()) self.assertEqual(b"foo bar", three.read()) four = GridOut(self.db.fs, file_document={}) with self.assertRaises(NoFile): four.name def test_write_file_like(self): one = GridIn(self.db.fs) one.write(b"hello world") one.close() two = GridOut(self.db.fs, one._id) three = GridIn(self.db.fs) three.write(two) three.close() four = GridOut(self.db.fs, three._id) self.assertEqual(b"hello world", four.read()) five = GridIn(self.db.fs, chunk_size=2) five.write(b"hello") buffer = StringIO(b" world") five.write(buffer) five.write(b" and mongodb") five.close() self.assertEqual(b"hello world and mongodb", GridOut(self.db.fs, five._id).read()) def test_write_lines(self): a = GridIn(self.db.fs) a.writelines([b"hello ", b"world"]) a.close() self.assertEqual(b"hello world", GridOut(self.db.fs, a._id).read()) def test_close(self): f = GridIn(self.db.fs) f.close() self.assertRaises(ValueError, f.write, "test") f.close() def test_multi_chunk_file(self): random_string = b'a' * (DEFAULT_CHUNK_SIZE + 1000) f = GridIn(self.db.fs) f.write(random_string) f.close() self.assertEqual(1, self.db.fs.files.find().count()) self.assertEqual(2, self.db.fs.chunks.find().count()) g = GridOut(self.db.fs, f._id) self.assertEqual(random_string, g.read()) def test_small_chunks(self): self.files = 0 self.chunks = 0 def helper(data): f = GridIn(self.db.fs, chunkSize=1) f.write(data) f.close() self.files += 1 self.chunks += len(data) self.assertEqual(self.files, self.db.fs.files.find().count()) self.assertEqual(self.chunks, self.db.fs.chunks.find().count()) g = GridOut(self.db.fs, f._id) self.assertEqual(data, g.read()) g = GridOut(self.db.fs, f._id) self.assertEqual(data, g.read(10) + g.read(10)) return True qcheck.check_unittest(self, helper, qcheck.gen_string(qcheck.gen_range(0, 20))) def test_seek(self): f = GridIn(self.db.fs, chunkSize=3) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(b"hello world", g.read()) g.seek(0) self.assertEqual(b"hello world", g.read()) g.seek(1) self.assertEqual(b"ello world", g.read()) self.assertRaises(IOError, g.seek, -1) g.seek(-3, _SEEK_END) self.assertEqual(b"rld", g.read()) g.seek(0, _SEEK_END) self.assertEqual(b"", g.read()) self.assertRaises(IOError, g.seek, -100, _SEEK_END) g.seek(3) g.seek(3, _SEEK_CUR) self.assertEqual(b"world", g.read()) self.assertRaises(IOError, g.seek, -100, _SEEK_CUR) def test_tell(self): f = GridIn(self.db.fs, chunkSize=3) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(0, g.tell()) g.read(0) self.assertEqual(0, g.tell()) g.read(1) self.assertEqual(1, g.tell()) g.read(2) self.assertEqual(3, g.tell()) g.read() self.assertEqual(g.length, g.tell()) def test_multiple_reads(self): f = GridIn(self.db.fs, chunkSize=3) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(b"he", g.read(2)) self.assertEqual(b"ll", g.read(2)) self.assertEqual(b"o ", g.read(2)) self.assertEqual(b"wo", g.read(2)) self.assertEqual(b"rl", g.read(2)) self.assertEqual(b"d", g.read(2)) self.assertEqual(b"", g.read(2)) def test_readline(self): f = GridIn(self.db.fs, chunkSize=5) f.write((b"""Hello world, How are you? Hope all is well. Bye""")) f.close() # Try read(), then readline(). g = GridOut(self.db.fs, f._id) self.assertEqual(b"H", g.read(1)) self.assertEqual(b"ello world,\n", g.readline()) self.assertEqual(b"How a", g.readline(5)) self.assertEqual(b"", g.readline(0)) self.assertEqual(b"re you?\n", g.readline()) self.assertEqual(b"Hope all is well.\n", g.readline(1000)) self.assertEqual(b"Bye", g.readline()) self.assertEqual(b"", g.readline()) # Try readline() first, then read(). g = GridOut(self.db.fs, f._id) self.assertEqual(b"He", g.readline(2)) self.assertEqual(b"l", g.read(1)) self.assertEqual(b"lo", g.readline(2)) self.assertEqual(b" world,\n", g.readline()) # Only readline(). g = GridOut(self.db.fs, f._id) self.assertEqual(b"H", g.readline(1)) self.assertEqual(b"e", g.readline(1)) self.assertEqual(b"llo world,\n", g.readline()) def test_iterator(self): f = GridIn(self.db.fs) f.close() g = GridOut(self.db.fs, f._id) self.assertEqual([], list(g)) f = GridIn(self.db.fs) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual([b"hello world"], list(g)) self.assertEqual(b"hello", g.read(5)) self.assertEqual([b"hello world"], list(g)) self.assertEqual(b" worl", g.read(5)) f = GridIn(self.db.fs, chunk_size=2) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual([b"he", b"ll", b"o ", b"wo", b"rl", b"d"], list(g)) def test_read_unaligned_buffer_size(self): in_data = (b"This is a text that doesn't " b"quite fit in a single 16-byte chunk.") f = GridIn(self.db.fs, chunkSize=16) f.write(in_data) f.close() g = GridOut(self.db.fs, f._id) out_data = b'' while 1: s = g.read(13) if not s: break out_data += s self.assertEqual(in_data, out_data) def test_readchunk(self): in_data = b'a' * 10 f = GridIn(self.db.fs, chunkSize=3) f.write(in_data) f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(3, len(g.readchunk())) self.assertEqual(2, len(g.read(2))) self.assertEqual(1, len(g.readchunk())) self.assertEqual(3, len(g.read(3))) self.assertEqual(1, len(g.readchunk())) self.assertEqual(0, len(g.readchunk())) def test_write_unicode(self): f = GridIn(self.db.fs) self.assertRaises(TypeError, f.write, u"foo") f = GridIn(self.db.fs, encoding="utf-8") f.write(u"foo") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(b"foo", g.read()) f = GridIn(self.db.fs, encoding="iso-8859-1") f.write(u"aé") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(u"aé".encode("iso-8859-1"), g.read()) def test_set_after_close(self): f = GridIn(self.db.fs, _id="foo", bar="baz") self.assertEqual("foo", f._id) self.assertEqual("baz", f.bar) self.assertRaises(AttributeError, getattr, f, "baz") self.assertRaises(AttributeError, getattr, f, "uploadDate") self.assertRaises(AttributeError, setattr, f, "_id", 5) f.bar = "foo" f.baz = 5 self.assertEqual("foo", f._id) self.assertEqual("foo", f.bar) self.assertEqual(5, f.baz) self.assertRaises(AttributeError, getattr, f, "uploadDate") f.close() self.assertEqual("foo", f._id) self.assertEqual("foo", f.bar) self.assertEqual(5, f.baz) self.assertTrue(f.uploadDate) self.assertRaises(AttributeError, setattr, f, "_id", 5) f.bar = "a" f.baz = "b" self.assertRaises(AttributeError, setattr, f, "upload_date", 5) g = GridOut(self.db.fs, f._id) self.assertEqual("a", g.bar) self.assertEqual("b", g.baz) # Versions 2.0.1 and older saved a _closed field for some reason. self.assertRaises(AttributeError, getattr, g, "_closed") def test_context_manager(self): contents = b"Imagine this is some important data..." with GridIn(self.db.fs, filename="important") as infile: infile.write(contents) with GridOut(self.db.fs, infile._id) as outfile: self.assertEqual(contents, outfile.read()) def test_prechunked_string(self): def write_me(s, chunk_size): buf = StringIO(s) infile = GridIn(self.db.fs) while True: to_write = buf.read(chunk_size) if to_write == b'': break infile.write(to_write) infile.close() buf.close() outfile = GridOut(self.db.fs, infile._id) data = outfile.read() self.assertEqual(s, data) s = b'x' * DEFAULT_CHUNK_SIZE * 4 # Test with default chunk size write_me(s, DEFAULT_CHUNK_SIZE) # Multiple write_me(s, DEFAULT_CHUNK_SIZE * 3) # Custom write_me(s, 262300) def test_grid_out_lazy_connect(self): fs = self.db.fs outfile = GridOut(fs, file_id=-1) self.assertRaises(NoFile, outfile.read) self.assertRaises(NoFile, getattr, outfile, 'filename') infile = GridIn(fs, filename=1) infile.close() outfile = GridOut(fs, infile._id) outfile.read() outfile.filename outfile = GridOut(fs, infile._id) outfile.readchunk() def test_grid_in_lazy_connect(self): client = MongoClient('badhost', connect=False, serverSelectionTimeoutMS=10) fs = client.db.fs infile = GridIn(fs, file_id=-1, chunk_size=1) self.assertRaises(ServerSelectionTimeoutError, infile.write, b'data') self.assertRaises(ServerSelectionTimeoutError, infile.close) def test_unacknowledged(self): # w=0 is prohibited. with self.assertRaises(ConfigurationError): GridIn(rs_or_single_client(w=0).pymongo_test.fs) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/utils.py0000644000076600000240000003710713245621354016141 0ustar shanestaff00000000000000# Copyright 2012-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for testing pymongo """ import contextlib import functools import os import struct import sys import threading import time import warnings from collections import defaultdict from functools import partial from pymongo import MongoClient, monitoring from pymongo.errors import AutoReconnect, OperationFailure from pymongo.server_selectors import (any_server_selector, writable_server_selector) from pymongo.write_concern import WriteConcern from test import (client_context, db_user, db_pwd) IMPOSSIBLE_WRITE_CONCERN = WriteConcern(w=1000) class WhiteListEventListener(monitoring.CommandListener): def __init__(self, *commands): self.commands = set(commands) self.results = defaultdict(list) def started(self, event): if event.command_name in self.commands: self.results['started'].append(event) def succeeded(self, event): if event.command_name in self.commands: self.results['succeeded'].append(event) def failed(self, event): if event.command_name in self.commands: self.results['failed'].append(event) class EventListener(monitoring.CommandListener): def __init__(self): self.results = defaultdict(list) def started(self, event): self.results['started'].append(event) def succeeded(self, event): self.results['succeeded'].append(event) def failed(self, event): self.results['failed'].append(event) class ServerAndTopologyEventListener(monitoring.ServerListener, monitoring.TopologyListener): """Listens to all events.""" def __init__(self): self.results = [] def opened(self, event): self.results.append(event) def description_changed(self, event): self.results.append(event) def closed(self, event): self.results.append(event) class HeartbeatEventListener(monitoring.ServerHeartbeatListener): """Listens to only server heartbeat events.""" def __init__(self): self.results = [] def started(self, event): self.results.append(event) def succeeded(self, event): self.results.append(event) def failed(self, event): self.results.append(event) def _connection_string(h, p, authenticate): if h.startswith("mongodb://"): return h elif client_context.auth_enabled and authenticate: return "mongodb://%s:%s@%s:%d" % (db_user, db_pwd, h, p) else: return "mongodb://%s:%d" % (h, p) def _mongo_client(host, port, authenticate=True, direct=False, **kwargs): """Create a new client over SSL/TLS if necessary.""" client_options = client_context.ssl_client_options.copy() if client_context.replica_set_name and not direct: client_options['replicaSet'] = client_context.replica_set_name client_options.update(kwargs) client = MongoClient(_connection_string(host, port, authenticate), port, **client_options) return client def single_client_noauth( h=client_context.host, p=client_context.port, **kwargs): """Make a direct connection. Don't authenticate.""" return _mongo_client(h, p, authenticate=False, direct=True, **kwargs) def single_client( h=client_context.host, p=client_context.port, **kwargs): """Make a direct connection, and authenticate if necessary.""" return _mongo_client(h, p, direct=True, **kwargs) def rs_client_noauth( h=client_context.host, p=client_context.port, **kwargs): """Connect to the replica set. Don't authenticate.""" return _mongo_client(h, p, authenticate=False, **kwargs) def rs_client( h=client_context.host, p=client_context.port, **kwargs): """Connect to the replica set and authenticate if necessary.""" return _mongo_client(h, p, **kwargs) def rs_or_single_client_noauth( h=client_context.host, p=client_context.port, **kwargs): """Connect to the replica set if there is one, otherwise the standalone. Like rs_or_single_client, but does not authenticate. """ return _mongo_client(h, p, authenticate=False, **kwargs) def rs_or_single_client( h=client_context.host, p=client_context.port, **kwargs): """Connect to the replica set if there is one, otherwise the standalone. Authenticates if necessary. """ return _mongo_client(h, p, **kwargs) def one(s): """Get one element of a set""" return next(iter(s)) def oid_generated_on_client(oid): """Is this process's PID in this ObjectId?""" pid_from_doc = struct.unpack(">H", oid.binary[7:9])[0] return (os.getpid() % 0xFFFF) == pid_from_doc def delay(sec): return '''function() { sleep(%f * 1000); return true; }''' % sec def get_command_line(client): command_line = client.admin.command('getCmdLineOpts') assert command_line['ok'] == 1, "getCmdLineOpts() failed" return command_line def server_started_with_option(client, cmdline_opt, config_opt): """Check if the server was started with a particular option. :Parameters: - `cmdline_opt`: The command line option (i.e. --nojournal) - `config_opt`: The config file option (i.e. nojournal) """ command_line = get_command_line(client) if 'parsed' in command_line: parsed = command_line['parsed'] if config_opt in parsed: return parsed[config_opt] argv = command_line['argv'] return cmdline_opt in argv def server_started_with_auth(client): try: command_line = get_command_line(client) except OperationFailure as e: msg = e.details.get('errmsg', '') if e.code == 13 or 'unauthorized' in msg or 'login' in msg: # Unauthorized. return True raise # MongoDB >= 2.0 if 'parsed' in command_line: parsed = command_line['parsed'] # MongoDB >= 2.6 if 'security' in parsed: security = parsed['security'] # >= rc3 if 'authorization' in security: return security['authorization'] == 'enabled' # < rc3 return security.get('auth', False) or bool(security.get('keyFile')) return parsed.get('auth', False) or bool(parsed.get('keyFile')) # Legacy argv = command_line['argv'] return '--auth' in argv or '--keyFile' in argv def server_started_with_nojournal(client): command_line = get_command_line(client) # MongoDB 2.6. if 'parsed' in command_line: parsed = command_line['parsed'] if 'storage' in parsed: storage = parsed['storage'] if 'journal' in storage: return not storage['journal']['enabled'] return server_started_with_option(client, '--nojournal', 'nojournal') def server_is_master_with_slave(client): command_line = get_command_line(client) if 'parsed' in command_line: return command_line['parsed'].get('master', False) return '--master' in command_line['argv'] def drop_collections(db): for coll in db.collection_names(): if not coll.startswith('system'): db.drop_collection(coll) def remove_all_users(db): db.command("dropAllUsersFromDatabase", 1, writeConcern={"w": client_context.w}) def joinall(threads): """Join threads with a 5-minute timeout, assert joins succeeded""" for t in threads: t.join(300) assert not t.isAlive(), "Thread %s hung" % t def connected(client): """Convenience to wait for a newly-constructed client to connect.""" with warnings.catch_warnings(): # Ignore warning that "ismaster" is always routed to primary even # if client's read preference isn't PRIMARY. warnings.simplefilter("ignore", UserWarning) client.admin.command('ismaster') # Force connection. return client def wait_until(predicate, success_description, timeout=10): """Wait up to 10 seconds (by default) for predicate to be true. E.g.: wait_until(lambda: client.primary == ('a', 1), 'connect to the primary') If the lambda-expression isn't true after 10 seconds, we raise AssertionError("Didn't ever connect to the primary"). Returns the predicate's first true value. """ start = time.time() while True: retval = predicate() if retval: return retval if time.time() - start > timeout: raise AssertionError("Didn't ever %s" % success_description) time.sleep(0.1) def is_mongos(client): res = client.admin.command('ismaster') return res.get('msg', '') == 'isdbgrid' def assertRaisesExactly(cls, fn, *args, **kwargs): """ Unlike the standard assertRaises, this checks that a function raises a specific class of exception, and not a subclass. E.g., check that MongoClient() raises ConnectionFailure but not its subclass, AutoReconnect. """ try: fn(*args, **kwargs) except Exception as e: assert e.__class__ == cls, "got %s, expected %s" % ( e.__class__.__name__, cls.__name__) else: raise AssertionError("%s not raised" % cls) @contextlib.contextmanager def _ignore_deprecations(): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) yield def ignore_deprecations(wrapped=None): """A context manager or a decorator.""" if wrapped: @functools.wraps(wrapped) def wrapper(*args, **kwargs): with _ignore_deprecations(): return wrapped(*args, **kwargs) return wrapper else: return _ignore_deprecations() class DeprecationFilter(object): def __init__(self, action="ignore"): """Start filtering deprecations.""" self.warn_context = warnings.catch_warnings() self.warn_context.__enter__() warnings.simplefilter(action, DeprecationWarning) def stop(self): """Stop filtering deprecations.""" self.warn_context.__exit__() self.warn_context = None def read_from_which_host( client, pref, tag_sets=None, ): """Read from a client with the given Read Preference. Return the 'host:port' which was read from. :Parameters: - `client`: A MongoClient - `mode`: A ReadPreference - `tag_sets`: List of dicts of tags for data-center-aware reads """ db = client.pymongo_test if isinstance(tag_sets, dict): tag_sets = [tag_sets] if tag_sets: tags = tag_sets or pref.tag_sets pref = pref.__class__(tags) db.read_preference = pref cursor = db.test.find() try: try: next(cursor) except StopIteration: # No documents in collection, that's fine pass return cursor.address except AutoReconnect: return None def assertReadFrom(testcase, client, member, *args, **kwargs): """Check that a query with the given mode and tag_sets reads from the expected replica-set member. :Parameters: - `testcase`: A unittest.TestCase - `client`: A MongoClient - `member`: A host:port expected to be used - `mode`: A ReadPreference - `tag_sets` (optional): List of dicts of tags for data-center-aware reads """ for _ in range(10): testcase.assertEqual(member, read_from_which_host(client, *args, **kwargs)) def assertReadFromAll(testcase, client, members, *args, **kwargs): """Check that a query with the given mode and tag_sets reads from all members in a set, and only members in that set. :Parameters: - `testcase`: A unittest.TestCase - `client`: A MongoClient - `members`: Sequence of host:port expected to be used - `mode`: A ReadPreference - `tag_sets` (optional): List of dicts of tags for data-center-aware reads """ members = set(members) used = set() for _ in range(100): used.add(read_from_which_host(client, *args, **kwargs)) testcase.assertEqual(members, used) def get_pool(client): """Get the standalone, primary, or mongos pool.""" topology = client._get_topology() server = topology.select_server(writable_server_selector) return server.pool def get_pools(client): """Get all pools.""" return [ server.pool for server in client._get_topology().select_servers(any_server_selector)] # Constants for run_threads and lazy_client_trial. NTRIALS = 5 NTHREADS = 10 def run_threads(collection, target): """Run a target function in many threads. target is a function taking a Collection and an integer. """ threads = [] for i in range(NTHREADS): bound_target = partial(target, collection, i) threads.append(threading.Thread(target=bound_target)) for t in threads: t.start() for t in threads: t.join(30) assert not t.isAlive() @contextlib.contextmanager def frequent_thread_switches(): """Make concurrency bugs more likely to manifest.""" interval = None if not sys.platform.startswith('java'): if hasattr(sys, 'getswitchinterval'): interval = sys.getswitchinterval() sys.setswitchinterval(1e-6) else: interval = sys.getcheckinterval() sys.setcheckinterval(1) try: yield finally: if not sys.platform.startswith('java'): if hasattr(sys, 'setswitchinterval'): sys.setswitchinterval(interval) else: sys.setcheckinterval(interval) def lazy_client_trial(reset, target, test, get_client): """Test concurrent operations on a lazily-connecting client. `reset` takes a collection and resets it for the next trial. `target` takes a lazily-connecting collection and an index from 0 to NTHREADS, and performs some operation, e.g. an insert. `test` takes the lazily-connecting collection and asserts a post-condition to prove `target` succeeded. """ collection = client_context.client.pymongo_test.test with frequent_thread_switches(): for i in range(NTRIALS): reset(collection) lazy_client = get_client() lazy_collection = lazy_client.pymongo_test.test run_threads(lazy_collection, target) test(lazy_collection) def gevent_monkey_patched(): """Check if gevent's monkey patching is active.""" # In Python 3.6 importing gevent.socket raises an ImportWarning. with warnings.catch_warnings(): warnings.simplefilter("ignore", ImportWarning) try: import socket import gevent.socket return socket.socket is gevent.socket.socket except ImportError: return False def eventlet_monkey_patched(): """Check if eventlet's monkey patching is active.""" try: import threading import eventlet return (threading.current_thread.__module__ == 'eventlet.green.threading') except ImportError: return False def is_greenthread_patched(): return gevent_monkey_patched() or eventlet_monkey_patched() pymongo-3.6.1/test/test_cursor_manager.py0000644000076600000240000000573413245621354021050 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the cursor_manager module.""" import sys import warnings sys.path[0:0] = [""] from pymongo.cursor_manager import CursorManager from pymongo.errors import CursorNotFound from pymongo.message import _CursorAddress from test import (client_context, client_knobs, unittest, IntegrationTest, SkipTest) from test.utils import rs_or_single_client, wait_until class TestCursorManager(IntegrationTest): @classmethod def setUpClass(cls): super(TestCursorManager, cls).setUpClass() cls.warn_context = warnings.catch_warnings() cls.warn_context.__enter__() warnings.simplefilter("ignore", DeprecationWarning) cls.collection = cls.db.test cls.collection.drop() # Ensure two batches. cls.collection.insert_many([{'_id': i} for i in range(200)]) @classmethod def tearDownClass(cls): cls.warn_context.__exit__() cls.warn_context = None cls.collection.drop() def test_cursor_manager_validation(self): with self.assertRaises(TypeError): client_context.client.set_cursor_manager(1) def test_cursor_manager(self): self.close_was_called = False test_case = self class CM(CursorManager): def __init__(self, client): super(CM, self).__init__(client) def close(self, cursor_id, address): test_case.close_was_called = True super(CM, self).close(cursor_id, address) with client_knobs(kill_cursor_frequency=0.01): client = rs_or_single_client(maxPoolSize=1) client.set_cursor_manager(CM) # Create a cursor on the same client so we're certain the getMore # is sent after the killCursors message. cursor = client.pymongo_test.test.find().batch_size(1) next(cursor) client.close_cursor( cursor.cursor_id, _CursorAddress(self.client.address, self.collection.full_name)) def raises_cursor_not_found(): try: next(cursor) return False except CursorNotFound: return True wait_until(raises_cursor_not_found, 'close cursor') self.assertTrue(self.close_was_called) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_gridfs_bucket.py0000644000076600000240000004474613245621354020662 0ustar shanestaff00000000000000# -*- coding: utf-8 -*- # # Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the gridfs package. """ import datetime import threading import time import gridfs from bson.binary import Binary from bson.objectid import ObjectId from bson.py3compat import StringIO, string_type from gridfs.errors import NoFile, CorruptGridFile from pymongo.errors import (ConfigurationError, ConnectionFailure, ServerSelectionTimeoutError) from pymongo.mongo_client import MongoClient from pymongo.read_preferences import ReadPreference from test import (client_context, unittest, IntegrationTest) from test.test_replica_set_client import TestReplicaSetClientBase from test.utils import (joinall, single_client, one, rs_client, rs_or_single_client) class JustWrite(threading.Thread): def __init__(self, gfs, num): threading.Thread.__init__(self) self.gfs = gfs self.num = num self.setDaemon(True) def run(self): for _ in range(self.num): file = self.gfs.open_upload_stream("test") file.write(b"hello") file.close() class JustRead(threading.Thread): def __init__(self, gfs, num, results): threading.Thread.__init__(self) self.gfs = gfs self.num = num self.results = results self.setDaemon(True) def run(self): for _ in range(self.num): file = self.gfs.open_download_stream_by_name("test") data = file.read() self.results.append(data) assert data == b"hello" class TestGridfs(IntegrationTest): @classmethod def setUpClass(cls): super(TestGridfs, cls).setUpClass() cls.fs = gridfs.GridFSBucket(cls.db) cls.alt = gridfs.GridFSBucket( cls.db, bucket_name="alt") def setUp(self): self.db.drop_collection("fs.files") self.db.drop_collection("fs.chunks") self.db.drop_collection("alt.files") self.db.drop_collection("alt.chunks") def test_basic(self): oid = self.fs.upload_from_stream("test_filename", b"hello world") self.assertEqual(b"hello world", self.fs.open_download_stream(oid).read()) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(1, self.db.fs.chunks.count()) self.fs.delete(oid) self.assertRaises(NoFile, self.fs.open_download_stream, oid) self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) def test_multi_chunk_delete(self): self.db.fs.drop() self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) gfs = gridfs.GridFSBucket(self.db) oid = gfs.upload_from_stream("test_filename", b"hello", chunk_size_bytes=1) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(5, self.db.fs.chunks.count()) gfs.delete(oid) self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) def test_empty_file(self): oid = self.fs.upload_from_stream("test_filename", b"") self.assertEqual(b"", self.fs.open_download_stream(oid).read()) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) raw = self.db.fs.files.find_one() self.assertEqual(0, raw["length"]) self.assertEqual(oid, raw["_id"]) self.assertTrue(isinstance(raw["uploadDate"], datetime.datetime)) self.assertEqual(255 * 1024, raw["chunkSize"]) self.assertTrue(isinstance(raw["md5"], string_type)) def test_corrupt_chunk(self): files_id = self.fs.upload_from_stream("test_filename", b'foobar') self.db.fs.chunks.update_one({'files_id': files_id}, {'$set': {'data': Binary(b'foo', 0)}}) try: out = self.fs.open_download_stream(files_id) self.assertRaises(CorruptGridFile, out.read) out = self.fs.open_download_stream(files_id) self.assertRaises(CorruptGridFile, out.readline) finally: self.fs.delete(files_id) def test_upload_ensures_index(self): # setUp has dropped collections. names = self.db.collection_names() self.assertFalse([name for name in names if name.startswith('fs')]) chunks = self.db.fs.chunks files = self.db.fs.files self.fs.upload_from_stream("filename", b"junk") self.assertTrue(any( info.get('key') == [('files_id', 1), ('n', 1)] for info in chunks.index_information().values())) self.assertTrue(any( info.get('key') == [('filename', 1), ('uploadDate', 1)] for info in files.index_information().values())) def test_alt_collection(self): oid = self.alt.upload_from_stream("test_filename", b"hello world") self.assertEqual(b"hello world", self.alt.open_download_stream(oid).read()) self.assertEqual(1, self.db.alt.files.count()) self.assertEqual(1, self.db.alt.chunks.count()) self.alt.delete(oid) self.assertRaises(NoFile, self.alt.open_download_stream, oid) self.assertEqual(0, self.db.alt.files.count()) self.assertEqual(0, self.db.alt.chunks.count()) self.assertRaises(NoFile, self.alt.open_download_stream, "foo") self.alt.upload_from_stream("foo", b"hello world") self.assertEqual(b"hello world", self.alt.open_download_stream_by_name("foo").read()) self.alt.upload_from_stream("mike", b"") self.alt.upload_from_stream("test", b"foo") self.alt.upload_from_stream("hello world", b"") self.assertEqual(set(["mike", "test", "hello world", "foo"]), set(k["filename"] for k in list( self.db.alt.files.find()))) def test_threaded_reads(self): self.fs.upload_from_stream("test", b"hello") threads = [] results = [] for i in range(10): threads.append(JustRead(self.fs, 10, results)) threads[i].start() joinall(threads) self.assertEqual( 100 * [b'hello'], results ) def test_threaded_writes(self): threads = [] for i in range(10): threads.append(JustWrite(self.fs, 10)) threads[i].start() joinall(threads) fstr = self.fs.open_download_stream_by_name("test") self.assertEqual(fstr.read(), b"hello") # Should have created 100 versions of 'test' file self.assertEqual( 100, self.db.fs.files.find({'filename': 'test'}).count() ) def test_get_last_version(self): one = self.fs.upload_from_stream("test", b"foo") time.sleep(0.01) two = self.fs.open_upload_stream("test") two.write(b"bar") two.close() time.sleep(0.01) two = two._id three = self.fs.upload_from_stream("test", b"baz") self.assertEqual(b"baz", self.fs.open_download_stream_by_name("test").read()) self.fs.delete(three) self.assertEqual(b"bar", self.fs.open_download_stream_by_name("test").read()) self.fs.delete(two) self.assertEqual(b"foo", self.fs.open_download_stream_by_name("test").read()) self.fs.delete(one) self.assertRaises(NoFile, self.fs.open_download_stream_by_name, "test") def test_get_version(self): self.fs.upload_from_stream("test", b"foo") time.sleep(0.01) self.fs.upload_from_stream("test", b"bar") time.sleep(0.01) self.fs.upload_from_stream("test", b"baz") time.sleep(0.01) self.assertEqual(b"foo", self.fs.open_download_stream_by_name( "test", revision=0).read()) self.assertEqual(b"bar", self.fs.open_download_stream_by_name( "test", revision=1).read()) self.assertEqual(b"baz", self.fs.open_download_stream_by_name( "test", revision=2).read()) self.assertEqual(b"baz", self.fs.open_download_stream_by_name( "test", revision=-1).read()) self.assertEqual(b"bar", self.fs.open_download_stream_by_name( "test", revision=-2).read()) self.assertEqual(b"foo", self.fs.open_download_stream_by_name( "test", revision=-3).read()) self.assertRaises(NoFile, self.fs.open_download_stream_by_name, "test", revision=3) self.assertRaises(NoFile, self.fs.open_download_stream_by_name, "test", revision=-4) def test_upload_from_stream(self): oid = self.fs.upload_from_stream("test_file", StringIO(b"hello world"), chunk_size_bytes=1) self.assertEqual(11, self.db.fs.chunks.count()) self.assertEqual(b"hello world", self.fs.open_download_stream(oid).read()) def test_upload_from_stream_with_id(self): oid = ObjectId() self.fs.upload_from_stream_with_id(oid, "test_file_custom_id", StringIO(b"custom id"), chunk_size_bytes=1) self.assertEqual(b"custom id", self.fs.open_download_stream(oid).read()) def test_open_upload_stream(self): gin = self.fs.open_upload_stream("from_stream") gin.write(b"from stream") gin.close() self.assertEqual(b"from stream", self.fs.open_download_stream(gin._id).read()) def test_open_upload_stream_with_id(self): oid = ObjectId() gin = self.fs.open_upload_stream_with_id(oid, "from_stream_custom_id") gin.write(b"from stream with custom id") gin.close() self.assertEqual(b"from stream with custom id", self.fs.open_download_stream(oid).read()) def test_missing_length_iter(self): # Test fix that guards against PHP-237 self.fs.upload_from_stream("empty", b"") doc = self.db.fs.files.find_one({"filename": "empty"}) doc.pop("length") self.db.fs.files.replace_one({"_id": doc["_id"]}, doc) fstr = self.fs.open_download_stream_by_name("empty") def iterate_file(grid_file): for _ in grid_file: pass return True self.assertTrue(iterate_file(fstr)) def test_gridfs_lazy_connect(self): client = MongoClient('badhost', connect=False, serverSelectionTimeoutMS=0) cdb = client.db gfs = gridfs.GridFSBucket(cdb) self.assertRaises(ServerSelectionTimeoutError, gfs.delete, 0) gfs = gridfs.GridFSBucket(cdb) self.assertRaises( ServerSelectionTimeoutError, gfs.upload_from_stream, "test", b"") # Still no connection. def test_gridfs_find(self): self.fs.upload_from_stream("two", b"test2") time.sleep(0.01) self.fs.upload_from_stream("two", b"test2+") time.sleep(0.01) self.fs.upload_from_stream("one", b"test1") time.sleep(0.01) self.fs.upload_from_stream("two", b"test2++") self.assertEqual(3, self.fs.find({"filename": "two"}).count()) self.assertEqual(4, self.fs.find({}).count()) cursor = self.fs.find( {}, no_cursor_timeout=False, sort=[("uploadDate", -1)], skip=1, limit=2) gout = next(cursor) self.assertEqual(b"test1", gout.read()) cursor.rewind() gout = next(cursor) self.assertEqual(b"test1", gout.read()) gout = next(cursor) self.assertEqual(b"test2+", gout.read()) self.assertRaises(StopIteration, cursor.__next__) cursor.close() self.assertRaises(TypeError, self.fs.find, {}, {"_id": True}) def test_grid_in_non_int_chunksize(self): # Lua, and perhaps other buggy GridFS clients, store size as a float. data = b'data' self.fs.upload_from_stream('f', data) self.db.fs.files.update_one({'filename': 'f'}, {'$set': {'chunkSize': 100.0}}) self.assertEqual(data, self.fs.open_download_stream_by_name('f').read()) def test_unacknowledged(self): # w=0 is prohibited. with self.assertRaises(ConfigurationError): gridfs.GridFSBucket(rs_or_single_client(w=0).pymongo_test) def test_rename(self): _id = self.fs.upload_from_stream("first_name", b'testing') self.assertEqual(b'testing', self.fs.open_download_stream_by_name( "first_name").read()) self.fs.rename(_id, "second_name") self.assertRaises(NoFile, self.fs.open_download_stream_by_name, "first_name") self.assertEqual(b"testing", self.fs.open_download_stream_by_name( "second_name").read()) def test_abort(self): gin = self.fs.open_upload_stream("test_filename", chunk_size_bytes=5) gin.write(b"test1") gin.write(b"test2") gin.write(b"test3") self.assertEqual(3, self.db.fs.chunks.count( {"files_id": gin._id})) gin.abort() self.assertTrue(gin.closed) self.assertRaises(ValueError, gin.write, b"test4") self.assertEqual(0, self.db.fs.chunks.count( {"files_id": gin._id})) def test_download_to_stream(self): file1 = StringIO(b"hello world") # Test with one chunk. oid = self.fs.upload_from_stream("one_chunk", file1) self.assertEqual(1, self.db.fs.chunks.count()) file2 = StringIO() self.fs.download_to_stream(oid, file2) file1.seek(0) file2.seek(0) self.assertEqual(file1.read(), file2.read()) # Test with many chunks. self.db.drop_collection("fs.files") self.db.drop_collection("fs.chunks") file1.seek(0) oid = self.fs.upload_from_stream("many_chunks", file1, chunk_size_bytes=1) self.assertEqual(11, self.db.fs.chunks.count()) file2 = StringIO() self.fs.download_to_stream(oid, file2) file1.seek(0) file2.seek(0) self.assertEqual(file1.read(), file2.read()) def test_download_to_stream_by_name(self): file1 = StringIO(b"hello world") # Test with one chunk. oid = self.fs.upload_from_stream("one_chunk", file1) self.assertEqual(1, self.db.fs.chunks.count()) file2 = StringIO() self.fs.download_to_stream_by_name("one_chunk", file2) file1.seek(0) file2.seek(0) self.assertEqual(file1.read(), file2.read()) # Test with many chunks. self.db.drop_collection("fs.files") self.db.drop_collection("fs.chunks") file1.seek(0) self.fs.upload_from_stream("many_chunks", file1, chunk_size_bytes=1) self.assertEqual(11, self.db.fs.chunks.count()) file2 = StringIO() self.fs.download_to_stream_by_name("many_chunks", file2) file1.seek(0) file2.seek(0) self.assertEqual(file1.read(), file2.read()) class TestGridfsBucketReplicaSet(TestReplicaSetClientBase): @classmethod @client_context.require_secondaries_count(1) def setUpClass(cls): super(TestGridfsBucketReplicaSet, cls).setUpClass() @classmethod def tearDownClass(cls): client_context.client.drop_database('gfsbucketreplica') def test_gridfs_replica_set(self): rsc = rs_client( w=self.w, read_preference=ReadPreference.SECONDARY) gfs = gridfs.GridFSBucket(rsc.gfsbucketreplica, 'gfsbucketreplicatest') oid = gfs.upload_from_stream("test_filename", b'foo') content = gfs.open_download_stream(oid).read() self.assertEqual(b'foo', content) def test_gridfs_secondary(self): primary_host, primary_port = self.primary primary_connection = single_client(primary_host, primary_port) secondary_host, secondary_port = one(self.secondaries) secondary_connection = single_client( secondary_host, secondary_port, read_preference=ReadPreference.SECONDARY) # Should detect it's connected to secondary and not attempt to # create index gfs = gridfs.GridFSBucket( secondary_connection.gfsbucketreplica, 'gfsbucketsecondarytest') # This won't detect secondary, raises error self.assertRaises(ConnectionFailure, gfs.upload_from_stream, "test_filename", b'foo') def test_gridfs_secondary_lazy(self): # Should detect it's connected to secondary and not attempt to # create index. secondary_host, secondary_port = one(self.secondaries) client = single_client( secondary_host, secondary_port, read_preference=ReadPreference.SECONDARY, connect=False) # Still no connection. gfs = gridfs.GridFSBucket( client.gfsbucketreplica, 'gfsbucketsecondarylazytest') # Connects, doesn't create index. self.assertRaises(NoFile, gfs.open_download_stream_by_name, "test_filename") self.assertRaises(ConnectionFailure, gfs.upload_from_stream, "test_filename", b'data') if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_legacy_api.py0000644000076600000240000026220013245666271020136 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test various legacy / deprecated API features.""" import itertools import sys import threading import time import uuid import warnings sys.path[0:0] = [""] from bson.binary import PYTHON_LEGACY, STANDARD from bson.code import Code from bson.codec_options import CodecOptions from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import string_type from bson.son import SON from pymongo import ASCENDING, DESCENDING from pymongo.database import Database from pymongo.common import partition_node from pymongo.errors import (BulkWriteError, ConfigurationError, CursorNotFound, DocumentTooLarge, DuplicateKeyError, InvalidDocument, InvalidOperation, OperationFailure, WriteConcernError, WTimeoutError) from pymongo.message import _CursorAddress from pymongo.son_manipulator import (AutoReference, NamespaceInjector, ObjectIdShuffler, SONManipulator) from pymongo.write_concern import WriteConcern from test import client_context, qcheck, unittest, SkipTest from test.test_client import IntegrationTest from test.test_bulk import BulkTestBase, BulkAuthorizationTestBase from test.utils import (DeprecationFilter, joinall, oid_generated_on_client, remove_all_users, rs_or_single_client, rs_or_single_client_noauth, single_client, wait_until) class TestDeprecations(IntegrationTest): @classmethod def setUpClass(cls): super(TestDeprecations, cls).setUpClass() cls.deprecation_filter = DeprecationFilter("error") @classmethod def tearDownClass(cls): cls.deprecation_filter.stop() def test_save_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.save({})) def test_insert_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.insert({})) def test_update_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.update({}, {})) def test_remove_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.remove({})) def test_find_and_modify_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.find_and_modify({'i': 5}, {})) def test_add_son_manipulator_deprecation(self): db = self.client.pymongo_test self.assertRaises(DeprecationWarning, lambda: db.add_son_manipulator(AutoReference(db))) def test_ensure_index_deprecation(self): try: self.assertRaises( DeprecationWarning, lambda: self.db.test.ensure_index('i')) finally: self.db.test.drop() class TestLegacy(IntegrationTest): @classmethod def setUpClass(cls): super(TestLegacy, cls).setUpClass() cls.w = client_context.w cls.deprecation_filter = DeprecationFilter() @classmethod def tearDownClass(cls): cls.deprecation_filter.stop() def test_insert_find_one(self): # Tests legacy insert. db = self.db db.test.drop() self.assertEqual(0, len(list(db.test.find()))) doc = {"hello": u"world"} _id = db.test.insert(doc) self.assertEqual(1, len(list(db.test.find()))) self.assertEqual(doc, db.test.find_one()) self.assertEqual(doc["_id"], _id) self.assertTrue(isinstance(_id, ObjectId)) doc_class = dict # Work around http://bugs.jython.org/issue1728 if (sys.platform.startswith('java') and sys.version_info[:3] >= (2, 5, 2)): doc_class = SON db = self.client.get_database( db.name, codec_options=CodecOptions(document_class=doc_class)) def remove_insert_find_one(doc): db.test.remove({}) db.test.insert(doc) # SON equality is order sensitive. return db.test.find_one() == doc.to_dict() qcheck.check_unittest(self, remove_insert_find_one, qcheck.gen_mongo_dict(3)) def test_generator_insert(self): # Only legacy insert currently supports insert from a generator. db = self.db db.test.remove({}) self.assertEqual(db.test.find().count(), 0) db.test.insert(({'a': i} for i in range(5)), manipulate=False) self.assertEqual(5, db.test.count()) db.test.remove({}) db.test.insert(({'a': i} for i in range(5)), manipulate=True) self.assertEqual(5, db.test.count()) db.test.remove({}) def test_insert_multiple(self): # Tests legacy insert. db = self.db db.drop_collection("test") doc1 = {"hello": u"world"} doc2 = {"hello": u"mike"} self.assertEqual(db.test.find().count(), 0) ids = db.test.insert([doc1, doc2]) self.assertEqual(db.test.find().count(), 2) self.assertEqual(doc1, db.test.find_one({"hello": u"world"})) self.assertEqual(doc2, db.test.find_one({"hello": u"mike"})) self.assertEqual(2, len(ids)) self.assertEqual(doc1["_id"], ids[0]) self.assertEqual(doc2["_id"], ids[1]) ids = db.test.insert([{"hello": 1}]) self.assertTrue(isinstance(ids, list)) self.assertEqual(1, len(ids)) self.assertRaises(InvalidOperation, db.test.insert, []) # Generator that raises StopIteration on first call to next(). self.assertRaises(InvalidOperation, db.test.insert, (i for i in [])) def test_insert_multiple_with_duplicate(self): # Tests legacy insert. db = self.db db.drop_collection("test_insert_multiple_with_duplicate") collection = db.test_insert_multiple_with_duplicate collection.create_index([('i', ASCENDING)], unique=True) # No error collection.insert([{'i': i} for i in range(5, 10)], w=0) wait_until(lambda: 5 == collection.count(), 'insert 5 documents') db.drop_collection("test_insert_multiple_with_duplicate") collection.create_index([('i', ASCENDING)], unique=True) # No error collection.insert([{'i': 1}] * 2, w=0) wait_until(lambda: 1 == collection.count(), 'insert 1 document') self.assertRaises( DuplicateKeyError, lambda: collection.insert([{'i': 2}] * 2), ) db.drop_collection("test_insert_multiple_with_duplicate") db = self.client.get_database( db.name, write_concern=WriteConcern(w=0)) collection = db.test_insert_multiple_with_duplicate collection.create_index([('i', ASCENDING)], unique=True) # No error. collection.insert([{'i': 1}] * 2) wait_until(lambda: 1 == collection.count(), 'insert 1 document') # Implied acknowledged. self.assertRaises( DuplicateKeyError, lambda: collection.insert([{'i': 2}] * 2, fsync=True), ) # Explicit acknowledged. self.assertRaises( DuplicateKeyError, lambda: collection.insert([{'i': 2}] * 2, w=1)) db.drop_collection("test_insert_multiple_with_duplicate") @client_context.require_replica_set def test_insert_prefers_write_errors(self): # Tests legacy insert. collection = self.db.test_insert_prefers_write_errors self.db.drop_collection(collection.name) collection.insert_one({'_id': 1}) large = 's' * 1024 * 1024 * 15 with self.assertRaises(DuplicateKeyError): collection.insert( [{'_id': 1, 's': large}, {'_id': 2, 's': large}]) self.assertEqual(1, collection.count()) with self.assertRaises(DuplicateKeyError): collection.insert( [{'_id': 1, 's': large}, {'_id': 2, 's': large}], continue_on_error=True) self.assertEqual(2, collection.count()) collection.delete_one({'_id': 2}) # A writeError followed by a writeConcernError should prefer to raise # the writeError. with self.assertRaises(DuplicateKeyError): collection.insert( [{'_id': 1, 's': large}, {'_id': 2, 's': large}], continue_on_error=True, w=len(client_context.nodes) + 10, wtimeout=1) self.assertEqual(2, collection.count()) collection.delete_many({}) with self.assertRaises(WriteConcernError): collection.insert( [{'_id': 1, 's': large}, {'_id': 2, 's': large}], continue_on_error=True, w=len(client_context.nodes) + 10, wtimeout=1) self.assertEqual(2, collection.count()) def test_insert_iterables(self): # Tests legacy insert. db = self.db self.assertRaises(TypeError, db.test.insert, 4) self.assertRaises(TypeError, db.test.insert, None) self.assertRaises(TypeError, db.test.insert, True) db.drop_collection("test") self.assertEqual(db.test.find().count(), 0) db.test.insert(({"hello": u"world"}, {"hello": u"world"})) self.assertEqual(db.test.find().count(), 2) db.drop_collection("test") self.assertEqual(db.test.find().count(), 0) db.test.insert(map(lambda x: {"hello": "world"}, itertools.repeat(None, 10))) self.assertEqual(db.test.find().count(), 10) def test_insert_manipulate_false(self): # Test two aspects of legacy insert with manipulate=False: # 1. The return value is None or [None] as appropriate. # 2. _id is not set on the passed-in document object. collection = self.db.test_insert_manipulate_false collection.drop() oid = ObjectId() doc = {'a': oid} try: # The return value is None. self.assertTrue(collection.insert(doc, manipulate=False) is None) # insert() shouldn't set _id on the passed-in document object. self.assertEqual({'a': oid}, doc) # Bulk insert. The return value is a list of None. self.assertEqual([None], collection.insert([{}], manipulate=False)) docs = [{}, {}] ids = collection.insert(docs, manipulate=False) self.assertEqual([None, None], ids) self.assertEqual([{}, {}], docs) finally: collection.drop() def test_continue_on_error(self): # Tests legacy insert. db = self.db db.drop_collection("test_continue_on_error") collection = db.test_continue_on_error oid = collection.insert({"one": 1}) self.assertEqual(1, collection.count()) docs = [] docs.append({"_id": oid, "two": 2}) # Duplicate _id. docs.append({"three": 3}) docs.append({"four": 4}) docs.append({"five": 5}) with self.assertRaises(DuplicateKeyError): collection.insert(docs, manipulate=False) self.assertEqual(1, collection.count()) with self.assertRaises(DuplicateKeyError): collection.insert(docs, manipulate=False, continue_on_error=True) self.assertEqual(4, collection.count()) collection.remove({}, w=client_context.w) oid = collection.insert({"_id": oid, "one": 1}, w=0) wait_until(lambda: 1 == collection.count(), 'insert 1 document') docs[0].pop("_id") docs[2]["_id"] = oid with self.assertRaises(DuplicateKeyError): collection.insert(docs, manipulate=False) self.assertEqual(3, collection.count()) collection.insert(docs, manipulate=False, continue_on_error=True, w=0) wait_until(lambda: 6 == collection.count(), 'insert 3 documents') def test_acknowledged_insert(self): # Tests legacy insert. db = self.db db.drop_collection("test_acknowledged_insert") collection = db.test_acknowledged_insert a = {"hello": "world"} collection.insert(a) collection.insert(a, w=0) self.assertRaises(OperationFailure, collection.insert, a) def test_insert_adds_id(self): # Tests legacy insert. doc = {"hello": "world"} self.db.test.insert(doc) self.assertTrue("_id" in doc) docs = [{"hello": "world"}, {"hello": "world"}] self.db.test.insert(docs) for doc in docs: self.assertTrue("_id" in doc) def test_insert_large_batch(self): # Tests legacy insert. db = self.client.test_insert_large_batch self.addCleanup(self.client.drop_database, 'test_insert_large_batch') max_bson_size = self.client.max_bson_size # Write commands are limited to 16MB + 16k per batch big_string = 'x' * int(max_bson_size / 2) # Batch insert that requires 2 batches. successful_insert = [{'x': big_string}, {'x': big_string}, {'x': big_string}, {'x': big_string}] db.collection_0.insert(successful_insert, w=1) self.assertEqual(4, db.collection_0.count()) db.collection_0.drop() # Test that inserts fail after first error. insert_second_fails = [{'_id': 'id0', 'x': big_string}, {'_id': 'id0', 'x': big_string}, {'_id': 'id1', 'x': big_string}, {'_id': 'id2', 'x': big_string}] with self.assertRaises(DuplicateKeyError): db.collection_1.insert(insert_second_fails) self.assertEqual(1, db.collection_1.count()) db.collection_1.drop() # 2 batches, 2nd insert fails, don't continue on error. self.assertTrue(db.collection_2.insert(insert_second_fails, w=0)) wait_until(lambda: 1 == db.collection_2.count(), 'insert 1 document', timeout=60) db.collection_2.drop() # 2 batches, ids of docs 0 and 1 are dupes, ids of docs 2 and 3 are # dupes. Acknowledged, continue on error. insert_two_failures = [{'_id': 'id0', 'x': big_string}, {'_id': 'id0', 'x': big_string}, {'_id': 'id1', 'x': big_string}, {'_id': 'id1', 'x': big_string}] with self.assertRaises(OperationFailure) as context: db.collection_3.insert(insert_two_failures, continue_on_error=True, w=1) self.assertIn('id1', str(context.exception)) # Only the first and third documents should be inserted. self.assertEqual(2, db.collection_3.count()) db.collection_3.drop() # 2 batches, 2 errors, unacknowledged, continue on error. db.collection_4.insert(insert_two_failures, continue_on_error=True, w=0) # Only the first and third documents are inserted. wait_until(lambda: 2 == db.collection_4.count(), 'insert 2 documents', timeout=60) db.collection_4.drop() def test_bad_dbref(self): # Requires the legacy API to test. c = self.db.test c.drop() # Incomplete DBRefs. self.assertRaises( InvalidDocument, c.insert_one, {'ref': {'$ref': 'collection'}}) self.assertRaises( InvalidDocument, c.insert_one, {'ref': {'$id': ObjectId()}}) ref_only = {'ref': {'$ref': 'collection'}} id_only = {'ref': {'$id': ObjectId()}} def test_update(self): # Tests legacy update. db = self.db db.drop_collection("test") id1 = db.test.save({"x": 5}) db.test.update({}, {"$inc": {"x": 1}}) self.assertEqual(db.test.find_one(id1)["x"], 6) id2 = db.test.save({"x": 1}) db.test.update({"x": 6}, {"$inc": {"x": 1}}) self.assertEqual(db.test.find_one(id1)["x"], 7) self.assertEqual(db.test.find_one(id2)["x"], 1) def test_update_manipulate(self): # Tests legacy update. db = self.db db.drop_collection("test") db.test.insert({'_id': 1}) db.test.update({'_id': 1}, {'a': 1}, manipulate=True) self.assertEqual( {'_id': 1, 'a': 1}, db.test.find_one()) class AddField(SONManipulator): def transform_incoming(self, son, dummy): son['field'] = 'value' return son db.add_son_manipulator(AddField()) db.test.update({'_id': 1}, {'a': 2}, manipulate=False) self.assertEqual( {'_id': 1, 'a': 2}, db.test.find_one()) db.test.update({'_id': 1}, {'a': 3}, manipulate=True) self.assertEqual( {'_id': 1, 'a': 3, 'field': 'value'}, db.test.find_one()) def test_update_nmodified(self): # Tests legacy update. db = self.db db.drop_collection("test") ismaster = self.client.admin.command('ismaster') used_write_commands = (ismaster.get("maxWireVersion", 0) > 1) db.test.insert({'_id': 1}) result = db.test.update({'_id': 1}, {'$set': {'x': 1}}) if used_write_commands: self.assertEqual(1, result['nModified']) else: self.assertFalse('nModified' in result) # x is already 1. result = db.test.update({'_id': 1}, {'$set': {'x': 1}}) if used_write_commands: self.assertEqual(0, result['nModified']) else: self.assertFalse('nModified' in result) def test_multi_update(self): # Tests legacy update. db = self.db db.drop_collection("test") db.test.save({"x": 4, "y": 3}) db.test.save({"x": 5, "y": 5}) db.test.save({"x": 4, "y": 4}) db.test.update({"x": 4}, {"$set": {"y": 5}}, multi=True) self.assertEqual(3, db.test.count()) for doc in db.test.find(): self.assertEqual(5, doc["y"]) self.assertEqual(2, db.test.update({"x": 4}, {"$set": {"y": 6}}, multi=True)["n"]) def test_upsert(self): # Tests legacy update. db = self.db db.drop_collection("test") db.test.update({"page": "/"}, {"$inc": {"count": 1}}, upsert=True) db.test.update({"page": "/"}, {"$inc": {"count": 1}}, upsert=True) self.assertEqual(1, db.test.count()) self.assertEqual(2, db.test.find_one()["count"]) def test_acknowledged_update(self): # Tests legacy update. db = self.db db.drop_collection("test_acknowledged_update") collection = db.test_acknowledged_update collection.create_index("x", unique=True) collection.insert({"x": 5}) _id = collection.insert({"x": 4}) self.assertEqual( None, collection.update({"_id": _id}, {"$inc": {"x": 1}}, w=0)) self.assertRaises(DuplicateKeyError, collection.update, {"_id": _id}, {"$inc": {"x": 1}}) self.assertEqual(1, collection.update({"_id": _id}, {"$inc": {"x": 2}})["n"]) self.assertEqual(0, collection.update({"_id": "foo"}, {"$inc": {"x": 2}})["n"]) db.drop_collection("test_acknowledged_update") def test_update_backward_compat(self): # MongoDB versions >= 2.6.0 don't return the updatedExisting field # and return upsert _id in an array subdocument. This test should # pass regardless of server version or type (mongod/s). # Tests legacy update. c = self.db.test c.drop() oid = ObjectId() res = c.update({'_id': oid}, {'$set': {'a': 'a'}}, upsert=True) self.assertFalse(res.get('updatedExisting')) self.assertEqual(oid, res.get('upserted')) res = c.update({'_id': oid}, {'$set': {'b': 'b'}}) self.assertTrue(res.get('updatedExisting')) def test_save(self): # Tests legacy save. self.db.drop_collection("test_save") collection = self.db.test_save # Save a doc with autogenerated id _id = collection.save({"hello": "world"}) self.assertEqual(collection.find_one()["_id"], _id) self.assertTrue(isinstance(_id, ObjectId)) # Save a doc with explicit id collection.save({"_id": "explicit_id", "hello": "bar"}) doc = collection.find_one({"_id": "explicit_id"}) self.assertEqual(doc['_id'], 'explicit_id') self.assertEqual(doc['hello'], 'bar') # Save docs with _id field already present (shouldn't create new docs) self.assertEqual(2, collection.count()) collection.save({'_id': _id, 'hello': 'world'}) self.assertEqual(2, collection.count()) collection.save({'_id': 'explicit_id', 'hello': 'baz'}) self.assertEqual(2, collection.count()) self.assertEqual( 'baz', collection.find_one({'_id': 'explicit_id'})['hello'] ) # Acknowledged mode. collection.create_index("hello", unique=True) # No exception, even though we duplicate the first doc's "hello" value collection.save({'_id': 'explicit_id', 'hello': 'world'}, w=0) self.assertRaises( DuplicateKeyError, collection.save, {'_id': 'explicit_id', 'hello': 'world'}) self.db.drop_collection("test") def test_save_with_invalid_key(self): if client_context.version.at_least(3, 5, 8): raise SkipTest("MongoDB >= 3.5.8 allows dotted fields in updates") # Tests legacy save. self.db.drop_collection("test") self.assertTrue(self.db.test.insert({"hello": "world"})) doc = self.db.test.find_one() doc['a.b'] = 'c' self.assertRaises(OperationFailure, self.db.test.save, doc) def test_acknowledged_save(self): # Tests legacy save. db = self.db db.drop_collection("test_acknowledged_save") collection = db.test_acknowledged_save collection.create_index("hello", unique=True) collection.save({"hello": "world"}) collection.save({"hello": "world"}, w=0) self.assertRaises(DuplicateKeyError, collection.save, {"hello": "world"}) db.drop_collection("test_acknowledged_save") def test_save_adds_id(self): # Tests legacy save. doc = {"hello": "jesse"} self.db.test.save(doc) self.assertTrue("_id" in doc) def test_save_returns_id(self): doc = {"hello": "jesse"} _id = self.db.test.save(doc) self.assertTrue(isinstance(_id, ObjectId)) self.assertEqual(_id, doc["_id"]) doc["hi"] = "bernie" _id = self.db.test.save(doc) self.assertTrue(isinstance(_id, ObjectId)) self.assertEqual(_id, doc["_id"]) def test_remove_one(self): # Tests legacy remove. self.db.test.remove() self.assertEqual(0, self.db.test.count()) self.db.test.insert({"x": 1}) self.db.test.insert({"y": 1}) self.db.test.insert({"z": 1}) self.assertEqual(3, self.db.test.count()) self.db.test.remove(multi=False) self.assertEqual(2, self.db.test.count()) self.db.test.remove() self.assertEqual(0, self.db.test.count()) def test_remove_all(self): # Tests legacy remove. self.db.test.remove() self.assertEqual(0, self.db.test.count()) self.db.test.insert({"x": 1}) self.db.test.insert({"y": 1}) self.assertEqual(2, self.db.test.count()) self.db.test.remove() self.assertEqual(0, self.db.test.count()) def test_remove_non_objectid(self): # Tests legacy remove. db = self.db db.drop_collection("test") db.test.insert_one({"_id": 5}) self.assertEqual(1, db.test.count()) db.test.remove(5) self.assertEqual(0, db.test.count()) def test_write_large_document(self): # Tests legacy insert, save, and update. max_size = self.db.client.max_bson_size half_size = int(max_size / 2) self.assertEqual(max_size, 16777216) self.assertRaises(OperationFailure, self.db.test.insert, {"foo": "x" * max_size}) self.assertRaises(OperationFailure, self.db.test.save, {"foo": "x" * max_size}) self.assertRaises(OperationFailure, self.db.test.insert, [{"x": 1}, {"foo": "x" * max_size}]) self.db.test.insert([{"foo": "x" * half_size}, {"foo": "x" * half_size}]) self.db.test.insert({"bar": "x"}) # Use w=0 here to test legacy doc size checking in all server versions self.assertRaises(DocumentTooLarge, self.db.test.update, {"bar": "x"}, {"bar": "x" * (max_size - 14)}, w=0) # This will pass with OP_UPDATE or the update command. self.db.test.update({"bar": "x"}, {"bar": "x" * (max_size - 32)}) def test_last_error_options(self): # Tests legacy write methods. self.db.test.save({"x": 1}, w=1, wtimeout=1) self.db.test.insert({"x": 1}, w=1, wtimeout=1) self.db.test.remove({"x": 1}, w=1, wtimeout=1) self.db.test.update({"x": 1}, {"y": 2}, w=1, wtimeout=1) if client_context.replica_set_name: # client_context.w is the number of hosts in the replica set w = client_context.w + 1 # MongoDB 2.8+ raises error code 100, CannotSatisfyWriteConcern, # if w > number of members. Older versions just time out after 1 ms # as if they had enough secondaries but some are lagging. They # return an error with 'wtimeout': True and no code. def wtimeout_err(f, *args, **kwargs): try: f(*args, **kwargs) except WTimeoutError as exc: self.assertIsNotNone(exc.details) except OperationFailure as exc: self.assertIsNotNone(exc.details) self.assertEqual(100, exc.code, "Unexpected error: %r" % exc) else: self.fail("%s should have failed" % f) coll = self.db.test wtimeout_err(coll.save, {"x": 1}, w=w, wtimeout=1) wtimeout_err(coll.insert, {"x": 1}, w=w, wtimeout=1) wtimeout_err(coll.update, {"x": 1}, {"y": 2}, w=w, wtimeout=1) wtimeout_err(coll.remove, {"x": 1}, w=w, wtimeout=1) # can't use fsync and j options together self.assertRaises(ConfigurationError, self.db.test.insert, {"_id": 1}, j=True, fsync=True) def test_find_and_modify(self): c = self.db.test c.drop() c.insert({'_id': 1, 'i': 1}) # Test that we raise DuplicateKeyError when appropriate. c.ensure_index('i', unique=True) self.assertRaises(DuplicateKeyError, c.find_and_modify, query={'i': 1, 'j': 1}, update={'$set': {'k': 1}}, upsert=True) c.drop_indexes() # Test correct findAndModify self.assertEqual({'_id': 1, 'i': 1}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual({'_id': 1, 'i': 3}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True)) self.assertEqual({'_id': 1, 'i': 3}, c.find_and_modify({'_id': 1}, remove=True)) self.assertEqual(None, c.find_one({'_id': 1})) self.assertEqual(None, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual(None, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, upsert=True)) self.assertEqual({'_id': 1, 'i': 2}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, upsert=True, new=True)) self.assertEqual({'_id': 1, 'i': 2}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, fields=['i'])) self.assertEqual({'_id': 1, 'i': 4}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, fields={'i': 1})) # Test with full_response=True. result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, upsert=True, full_response=True, fields={'i': 1}) self.assertEqual({'_id': 1, 'i': 5}, result["value"]) self.assertEqual(True, result["lastErrorObject"]["updatedExisting"]) result = c.find_and_modify({'_id': 2}, {'$inc': {'i': 1}}, new=True, upsert=True, full_response=True, fields={'i': 1}) self.assertEqual({'_id': 2, 'i': 1}, result["value"]) self.assertEqual(False, result["lastErrorObject"]["updatedExisting"]) class ExtendedDict(dict): pass result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, fields={'i': 1}) self.assertFalse(isinstance(result, ExtendedDict)) c = self.db.get_collection( "test", codec_options=CodecOptions(document_class=ExtendedDict)) result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, fields={'i': 1}) self.assertTrue(isinstance(result, ExtendedDict)) def test_find_and_modify_with_sort(self): c = self.db.test c.drop() for j in range(5): c.insert({'j': j, 'i': 0}) sort = {'j': DESCENDING} self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = {'j': ASCENDING} self.assertEqual(0, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = [('j', DESCENDING)] self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = [('j', ASCENDING)] self.assertEqual(0, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = SON([('j', DESCENDING)]) self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = SON([('j', ASCENDING)]) self.assertEqual(0, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) try: from collections import OrderedDict sort = OrderedDict([('j', DESCENDING)]) self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = OrderedDict([('j', ASCENDING)]) self.assertEqual(0, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) except ImportError: pass # Test that a standard dict with two keys is rejected. sort = {'j': DESCENDING, 'foo': DESCENDING} self.assertRaises(TypeError, c.find_and_modify, {}, {'$inc': {'i': 1}}, sort=sort) def test_find_and_modify_with_manipulator(self): class AddCollectionNameManipulator(SONManipulator): def will_copy(self): return True def transform_incoming(self, son, dummy): copy = SON(son) if 'collection' in copy: del copy['collection'] return copy def transform_outgoing(self, son, collection): copy = SON(son) copy['collection'] = collection.name return copy db = self.client.pymongo_test db.add_son_manipulator(AddCollectionNameManipulator()) c = db.test c.drop() c.insert({'_id': 1, 'i': 1}) # Test correct findAndModify # With manipulators self.assertEqual({'_id': 1, 'i': 1, 'collection': 'test'}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, manipulate=True)) self.assertEqual({'_id': 1, 'i': 3, 'collection': 'test'}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, manipulate=True)) # With out manipulators self.assertEqual({'_id': 1, 'i': 3}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual({'_id': 1, 'i': 5}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True)) def test_group(self): db = self.db db.drop_collection("test") self.assertEqual([], db.test.group([], {}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) db.test.insert_many([{"a": 2}, {"b": 5}, {"a": 1}]) self.assertEqual([{"count": 3}], db.test.group([], {}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) self.assertEqual([{"count": 1}], db.test.group([], {"a": {"$gt": 1}}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) db.test.insert_one({"a": 2, "b": 3}) self.assertEqual([{"a": 2, "count": 2}, {"a": None, "count": 1}, {"a": 1, "count": 1}], db.test.group(["a"], {}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) # modifying finalize self.assertEqual([{"a": 2, "count": 3}, {"a": None, "count": 2}, {"a": 1, "count": 2}], db.test.group(["a"], {}, {"count": 0}, "function (obj, prev) " "{ prev.count++; }", "function (obj) { obj.count++; }")) # returning finalize self.assertEqual([2, 1, 1], db.test.group(["a"], {}, {"count": 0}, "function (obj, prev) " "{ prev.count++; }", "function (obj) { return obj.count; }")) # keyf self.assertEqual([2, 2], db.test.group("function (obj) { if (obj.a == 2) " "{ return {a: true} }; " "return {b: true}; }", {}, {"count": 0}, "function (obj, prev) " "{ prev.count++; }", "function (obj) { return obj.count; }")) # no key self.assertEqual([{"count": 4}], db.test.group(None, {}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) self.assertRaises(OperationFailure, db.test.group, [], {}, {}, "5 ++ 5") def test_group_with_scope(self): db = self.db db.drop_collection("test") db.test.insert_many([{"a": 1}, {"b": 1}]) reduce_function = "function (obj, prev) { prev.count += inc_value; }" self.assertEqual(2, db.test.group([], {}, {"count": 0}, Code(reduce_function, {"inc_value": 1}))[0]['count']) self.assertEqual(4, db.test.group([], {}, {"count": 0}, Code(reduce_function, {"inc_value": 2}))[0]['count']) self.assertEqual(1, db.test.group([], {}, {"count": 0}, Code(reduce_function, {"inc_value": 0.5}))[0]['count']) self.assertEqual(2, db.test.group( [], {}, {"count": 0}, Code(reduce_function, {"inc_value": 1}))[0]['count']) self.assertEqual(4, db.test.group( [], {}, {"count": 0}, Code(reduce_function, {"inc_value": 2}))[0]['count']) self.assertEqual(1, db.test.group( [], {}, {"count": 0}, Code(reduce_function, {"inc_value": 0.5}))[0]['count']) def test_group_uuid_representation(self): db = self.db coll = db.uuid coll.drop() uu = uuid.uuid4() coll.insert_one({"_id": uu, "a": 2}) coll.insert_one({"_id": uuid.uuid4(), "a": 1}) reduce = "function (obj, prev) { prev.count++; }" coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) self.assertEqual([], coll.group([], {"_id": uu}, {"count": 0}, reduce)) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) self.assertEqual([{"count": 1}], coll.group([], {"_id": uu}, {"count": 0}, reduce)) def test_last_status(self): # Tests many legacy API elements. # We must call getlasterror on same socket as the last operation. db = rs_or_single_client(maxPoolSize=1).pymongo_test collection = db.test_last_status collection.remove({}) collection.save({"i": 1}) collection.update({"i": 1}, {"$set": {"i": 2}}, w=0) self.assertTrue(db.last_status()["updatedExisting"]) collection.update({"i": 1}, {"$set": {"i": 500}}, w=0) self.assertFalse(db.last_status()["updatedExisting"]) def test_auto_ref_and_deref(self): # Legacy API. db = self.client.pymongo_test db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.test.a.remove({}) db.test.b.remove({}) db.test.c.remove({}) a = {"hello": u"world"} db.test.a.save(a) b = {"test": a} db.test.b.save(b) c = {"another test": b} db.test.c.save(c) a["hello"] = "mike" db.test.a.save(a) self.assertEqual(db.test.a.find_one(), a) self.assertEqual(db.test.b.find_one()["test"], a) self.assertEqual(db.test.c.find_one()["another test"]["test"], a) self.assertEqual(db.test.b.find_one(), b) self.assertEqual(db.test.c.find_one()["another test"], b) self.assertEqual(db.test.c.find_one(), c) def test_auto_ref_and_deref_list(self): # Legacy API. db = self.client.pymongo_test db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.drop_collection("users") db.drop_collection("messages") message_1 = {"title": "foo"} db.messages.save(message_1) message_2 = {"title": "bar"} db.messages.save(message_2) user = {"messages": [message_1, message_2]} db.users.save(user) db.messages.update(message_1, {"title": "buzz"}) self.assertEqual("buzz", db.users.find_one()["messages"][0]["title"]) self.assertEqual("bar", db.users.find_one()["messages"][1]["title"]) def test_object_to_dict_transformer(self): # PYTHON-709: Some users rely on their custom SONManipulators to run # before any other checks, so they can insert non-dict objects and # have them dictified before the _id is inserted or any other # processing. # Tests legacy API elements. class Thing(object): def __init__(self, value): self.value = value class ThingTransformer(SONManipulator): def transform_incoming(self, thing, dummy): return {'value': thing.value} db = self.client.foo db.add_son_manipulator(ThingTransformer()) t = Thing('value') db.test.remove() db.test.insert([t]) out = db.test.find_one() self.assertEqual('value', out.get('value')) def test_son_manipulator_outgoing(self): class Thing(object): def __init__(self, value): self.value = value class ThingTransformer(SONManipulator): def transform_outgoing(self, doc, collection): # We don't want this applied to the command return # value in pymongo.cursor.Cursor. if 'value' in doc: return Thing(doc['value']) return doc db = self.client.foo db.add_son_manipulator(ThingTransformer()) db.test.delete_many({}) db.test.insert_one({'value': 'value'}) out = db.test.find_one() self.assertTrue(isinstance(out, Thing)) self.assertEqual('value', out.value) out = next(db.test.aggregate([], cursor={})) self.assertTrue(isinstance(out, Thing)) self.assertEqual('value', out.value) def test_son_manipulator_inheritance(self): # Tests legacy API elements. class Thing(object): def __init__(self, value): self.value = value class ThingTransformer(SONManipulator): def transform_incoming(self, thing, dummy): return {'value': thing.value} def transform_outgoing(self, son, dummy): return Thing(son['value']) class Child(ThingTransformer): pass db = self.client.foo db.add_son_manipulator(Child()) t = Thing('value') db.test.remove() db.test.insert([t]) out = db.test.find_one() self.assertTrue(isinstance(out, Thing)) self.assertEqual('value', out.value) def test_disabling_manipulators(self): class IncByTwo(SONManipulator): def transform_outgoing(self, son, collection): if 'foo' in son: son['foo'] += 2 return son db = self.client.pymongo_test db.add_son_manipulator(IncByTwo()) c = db.test c.drop() c.insert({'foo': 0}) self.assertEqual(2, c.find_one()['foo']) self.assertEqual(0, c.find_one(manipulate=False)['foo']) self.assertEqual(2, c.find_one(manipulate=True)['foo']) c.drop() def test_manipulator_properties(self): db = self.client.foo self.assertEqual([], db.incoming_manipulators) self.assertEqual([], db.incoming_copying_manipulators) self.assertEqual([], db.outgoing_manipulators) self.assertEqual([], db.outgoing_copying_manipulators) db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.add_son_manipulator(ObjectIdShuffler()) self.assertEqual(1, len(db.incoming_manipulators)) self.assertEqual(db.incoming_manipulators, ['NamespaceInjector']) self.assertEqual(2, len(db.incoming_copying_manipulators)) for name in db.incoming_copying_manipulators: self.assertTrue(name in ('ObjectIdShuffler', 'AutoReference')) self.assertEqual([], db.outgoing_manipulators) self.assertEqual(['AutoReference'], db.outgoing_copying_manipulators) def test_ensure_index(self): db = self.db self.assertRaises(TypeError, db.test.ensure_index, {"hello": 1}) self.assertRaises(TypeError, db.test.ensure_index, {"hello": 1}, cache_for='foo') db.test.drop_indexes() self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.test.drop_indexes() self.assertEqual("foo", db.test.ensure_index("goodbye", name="foo")) self.assertEqual(None, db.test.ensure_index("goodbye", name="foo")) db.test.drop_indexes() self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.test.drop_index("goodbye_1") self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.drop_collection("test") self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.test.drop_index("goodbye_1") self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.test.drop_index("goodbye_1") self.assertEqual("goodbye_1", db.test.ensure_index("goodbye", cache_for=1)) time.sleep(1.2) self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) # Make sure the expiration time is updated. self.assertEqual(None, db.test.ensure_index("goodbye")) # Clean up indexes for later tests db.test.drop_indexes() def test_ensure_index_threaded(self): coll = self.db.threaded_index_creation index_docs = [] class Indexer(threading.Thread): def run(self): coll.ensure_index('foo0') coll.ensure_index('foo1') coll.ensure_index('foo2') index_docs.append(coll.index_information()) try: threads = [] for _ in range(10): t = Indexer() t.setDaemon(True) threads.append(t) for thread in threads: thread.start() joinall(threads) first = index_docs[0] for index_doc in index_docs[1:]: self.assertEqual(index_doc, first) finally: coll.drop() def test_ensure_purge_index_threaded(self): coll = self.db.threaded_index_creation class Indexer(threading.Thread): def run(self): coll.ensure_index('foo') try: coll.drop_index('foo') except OperationFailure: # The index may have already been dropped. pass coll.ensure_index('foo') coll.drop_indexes() coll.create_index('foo') try: threads = [] for _ in range(10): t = Indexer() t.setDaemon(True) threads.append(t) for thread in threads: thread.start() joinall(threads) self.assertTrue('foo_1' in coll.index_information()) finally: coll.drop() def test_ensure_unique_index_threaded(self): coll = self.db.test_unique_threaded coll.drop() coll.insert_many([{'foo': i} for i in range(10000)]) class Indexer(threading.Thread): def run(self): try: coll.ensure_index('foo', unique=True) coll.insert_one({'foo': 'bar'}) coll.insert_one({'foo': 'bar'}) except OperationFailure: pass threads = [] for _ in range(10): t = Indexer() t.setDaemon(True) threads.append(t) for i in range(10): threads[i].start() joinall(threads) self.assertEqual(10001, coll.count()) coll.drop() def test_kill_cursors_with_cursoraddress(self): coll = self.client.pymongo_test.test coll.drop() coll.insert_many([{'_id': i} for i in range(200)]) cursor = coll.find().batch_size(1) next(cursor) self.client.kill_cursors( [cursor.cursor_id], _CursorAddress(self.client.address, coll.full_name)) # Prevent killcursors from reaching the server while a getmore is in # progress -- the server logs "Assertion: 16089:Cannot kill active # cursor." time.sleep(2) def raises_cursor_not_found(): try: next(cursor) return False except CursorNotFound: return True wait_until(raises_cursor_not_found, 'close cursor') def test_kill_cursors_with_tuple(self): if (client_context.version[:2] == (3, 6) and client_context.auth_enabled): raise SkipTest("SERVER-33553") coll = self.client.pymongo_test.test coll.drop() coll.insert_many([{'_id': i} for i in range(200)]) cursor = coll.find().batch_size(1) next(cursor) self.client.kill_cursors( [cursor.cursor_id], self.client.address) # Prevent killcursors from reaching the server while a getmore is in # progress -- the server logs "Assertion: 16089:Cannot kill active # cursor." time.sleep(2) def raises_cursor_not_found(): try: next(cursor) return False except CursorNotFound: return True wait_until(raises_cursor_not_found, 'close cursor') def test_get_default_database(self): c = rs_or_single_client("mongodb://%s:%d/foo" % (client_context.host, client_context.port), connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database()) def test_get_default_database_error(self): # URI with no database. c = rs_or_single_client("mongodb://%s:%d/" % (client_context.host, client_context.port), connect=False) self.assertRaises(ConfigurationError, c.get_default_database) def test_get_default_database_with_authsource(self): # Ensure we distinguish database name from authSource. uri = "mongodb://%s:%d/foo?authSource=src" % ( client_context.host, client_context.port) c = rs_or_single_client(uri, connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database()) class TestLegacyBulk(BulkTestBase): @classmethod def setUpClass(cls): super(TestLegacyBulk, cls).setUpClass() cls.deprecation_filter = DeprecationFilter() @classmethod def tearDownClass(cls): cls.deprecation_filter.stop() def test_empty(self): bulk = self.coll.initialize_ordered_bulk_op() self.assertRaises(InvalidOperation, bulk.execute) def test_find(self): # find() requires a selector. bulk = self.coll.initialize_ordered_bulk_op() self.assertRaises(TypeError, bulk.find) self.assertRaises(TypeError, bulk.find, 'foo') # No error. bulk.find({}) @client_context.require_version_min(3, 1, 9, -1) def test_bypass_document_validation_bulk_op(self): # Test insert self.coll.insert_one({"z": 0}) self.db.command(SON([("collMod", "test"), ("validator", {"z": {"$gte": 0}})])) bulk = self.coll.initialize_ordered_bulk_op( bypass_document_validation=False) bulk.insert({"z": -1}) # error self.assertRaises(BulkWriteError, bulk.execute) self.assertEqual(0, self.coll.count({"z": -1})) bulk = self.coll.initialize_ordered_bulk_op( bypass_document_validation=True) bulk.insert({"z": -1}) bulk.execute() self.assertEqual(1, self.coll.count({"z": -1})) self.coll.insert_one({"z": 0}) self.db.command(SON([("collMod", "test"), ("validator", {"z": {"$gte": 0}})])) bulk = self.coll.initialize_unordered_bulk_op( bypass_document_validation=False) bulk.insert({"z": -1}) # error self.assertRaises(BulkWriteError, bulk.execute) self.assertEqual(1, self.coll.count({"z": -1})) bulk = self.coll.initialize_unordered_bulk_op( bypass_document_validation=True) bulk.insert({"z": -1}) bulk.execute() self.assertEqual(2, self.coll.count({"z": -1})) self.coll.drop() def test_insert(self): expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 1, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } bulk = self.coll.initialize_ordered_bulk_op() self.assertRaises(TypeError, bulk.insert, 1) # find() before insert() is prohibited. self.assertRaises(AttributeError, lambda: bulk.find({}).insert({})) # We don't allow multiple documents per call. self.assertRaises(TypeError, bulk.insert, [{}, {}]) self.assertRaises(TypeError, bulk.insert, ({} for _ in range(2))) bulk.insert({}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(1, self.coll.count()) doc = self.coll.find_one() self.assertTrue(oid_generated_on_client(doc['_id'])) bulk = self.coll.initialize_unordered_bulk_op() bulk.insert({}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(2, self.coll.count()) def test_insert_check_keys(self): bulk = self.coll.initialize_ordered_bulk_op() bulk.insert({'$dollar': 1}) self.assertRaises(InvalidDocument, bulk.execute) bulk = self.coll.initialize_ordered_bulk_op() bulk.insert({'a.b': 1}) self.assertRaises(InvalidDocument, bulk.execute) def test_update(self): expected = { 'nMatched': 2, 'nModified': 2, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_ordered_bulk_op() # update() requires find() first. self.assertRaises( AttributeError, lambda: bulk.update({'$set': {'x': 1}})) self.assertRaises(TypeError, bulk.find({}).update, 1) self.assertRaises(ValueError, bulk.find({}).update, {}) # All fields must be $-operators. self.assertRaises(ValueError, bulk.find({}).update, {'foo': 'bar'}) bulk.find({}).update({'$set': {'foo': 'bar'}}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'foo': 'bar'}).count(), 2) # All fields must be $-operators -- validated server-side. bulk = self.coll.initialize_ordered_bulk_op() updates = SON([('$set', {'x': 1}), ('y', 1)]) bulk.find({}).update(updates) self.assertRaises(BulkWriteError, bulk.execute) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).update({'$set': {'bim': 'baz'}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 2, 'nModified': 2, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.find({'bim': 'baz'}).count(), 2) self.coll.insert_one({'x': 1}) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({'x': 1}).update({'$set': {'x': 42}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(1, self.coll.find({'x': 42}).count()) # Second time, x is already 42 so nModified is 0. bulk = self.coll.initialize_unordered_bulk_op() bulk.find({'x': 42}).update({'$set': {'x': 42}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) def test_update_one(self): expected = { 'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_ordered_bulk_op() # update_one() requires find() first. self.assertRaises( AttributeError, lambda: bulk.update_one({'$set': {'x': 1}})) self.assertRaises(TypeError, bulk.find({}).update_one, 1) self.assertRaises(ValueError, bulk.find({}).update_one, {}) self.assertRaises(ValueError, bulk.find({}).update_one, {'foo': 'bar'}) bulk.find({}).update_one({'$set': {'foo': 'bar'}}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'foo': 'bar'}).count(), 1) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).update_one({'$set': {'bim': 'baz'}}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'bim': 'baz'}).count(), 1) # All fields must be $-operators -- validated server-side. bulk = self.coll.initialize_ordered_bulk_op() updates = SON([('$set', {'x': 1}), ('y', 1)]) bulk.find({}).update_one(updates) self.assertRaises(BulkWriteError, bulk.execute) def test_replace_one(self): expected = { 'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_ordered_bulk_op() self.assertRaises(TypeError, bulk.find({}).replace_one, 1) self.assertRaises(ValueError, bulk.find({}).replace_one, {'$set': {'foo': 'bar'}}) bulk.find({}).replace_one({'foo': 'bar'}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'foo': 'bar'}).count(), 1) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).replace_one({'bim': 'baz'}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'bim': 'baz'}).count(), 1) def test_remove(self): # Test removing all documents, ordered. expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_ordered_bulk_op() # remove() must be preceded by find(). self.assertRaises(AttributeError, lambda: bulk.remove()) bulk.find({}).remove() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.count(), 0) # Test removing some documents, ordered. self.coll.insert_many([{}, {'x': 1}, {}, {'x': 1}]) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({'x': 1}).remove() result = bulk.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.count(), 2) self.coll.delete_many({}) # Test removing all documents, unordered. self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).remove() result = bulk.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) # Test removing some documents, unordered. self.assertEqual(self.coll.count(), 0) self.coll.insert_many([{}, {'x': 1}, {}, {'x': 1}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({'x': 1}).remove() result = bulk.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.count(), 2) self.coll.delete_many({}) def test_remove_one(self): bulk = self.coll.initialize_ordered_bulk_op() # remove_one() must be preceded by find(). self.assertRaises(AttributeError, lambda: bulk.remove_one()) # Test removing one document, empty selector. # First ordered, then unordered. self.coll.insert_many([{}, {}]) expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 1, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } bulk.find({}).remove_one() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.count(), 1) self.coll.insert_one({}) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).remove_one() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.count(), 1) # Test removing one document, with a selector. # First ordered, then unordered. self.coll.insert_one({'x': 1}) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({'x': 1}).remove_one() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual([{}], list(self.coll.find({}, {'_id': False}))) self.coll.insert_one({'x': 1}) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({'x': 1}).remove_one() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual([{}], list(self.coll.find({}, {'_id': False}))) def test_upsert(self): bulk = self.coll.initialize_ordered_bulk_op() # upsert() requires find() first. self.assertRaises( AttributeError, lambda: bulk.upsert()) expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 1, 'nInserted': 0, 'nRemoved': 0, 'upserted': [{'index': 0, '_id': '...'}] } bulk.find({}).upsert().replace_one({'foo': 'bar'}) result = bulk.execute() self.assertEqualResponse(expected, result) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({}).upsert().update_one({'$set': {'bim': 'baz'}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.find({'bim': 'baz'}).count(), 1) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({}).upsert().update({'$set': {'bim': 'bop'}}) # Non-upsert, no matches. bulk.find({'x': 1}).update({'$set': {'x': 2}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.find({'bim': 'bop'}).count(), 1) self.assertEqual(self.coll.find({'x': 2}).count(), 0) def test_upsert_large(self): big = 'a' * (client_context.client.max_bson_size - 37) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({'x': 1}).upsert().update({'$set': {'s': big}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 1, 'nInserted': 0, 'nRemoved': 0, 'upserted': [{'index': 0, '_id': '...'}]}, result) self.assertEqual(1, self.coll.find({'x': 1}).count()) def test_client_generated_upsert_id(self): batch = self.coll.initialize_ordered_bulk_op() batch.find({'_id': 0}).upsert().update_one({'$set': {'a': 0}}) batch.find({'a': 1}).upsert().replace_one({'_id': 1}) # This is just here to make the counts right in all cases. batch.find({'_id': 2}).upsert().replace_one({'_id': 2}) result = batch.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 3, 'nInserted': 0, 'nRemoved': 0, 'upserted': [{'index': 0, '_id': 0}, {'index': 1, '_id': 1}, {'index': 2, '_id': 2}]}, result) def test_single_ordered_batch(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 1}).update_one({'$set': {'b': 1}}) batch.find({'a': 2}).upsert().update_one({'$set': {'b': 2}}) batch.insert({'a': 3}) batch.find({'a': 3}).remove() result = batch.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 1, 'nInserted': 2, 'nRemoved': 1, 'upserted': [{'index': 2, '_id': '...'}]}, result) def test_single_error_ordered_batch(self): self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) batch.insert({'b': 3, 'a': 2}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 1, 'nRemoved': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [ {'index': 1, 'code': 11000, 'errmsg': '...', 'op': {'q': {'b': 2}, 'u': {'$set': {'a': 1}}, 'multi': False, 'upsert': True}}]}, result) def test_multiple_error_ordered_batch(self): self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) batch.find({'b': 3}).upsert().update_one({'$set': {'a': 2}}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) batch.insert({'b': 4, 'a': 3}) batch.insert({'b': 5, 'a': 1}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 1, 'nRemoved': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [ {'index': 1, 'code': 11000, 'errmsg': '...', 'op': {'q': {'b': 2}, 'u': {'$set': {'a': 1}}, 'multi': False, 'upsert': True}}]}, result) def test_single_unordered_batch(self): batch = self.coll.initialize_unordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 1}).update_one({'$set': {'b': 1}}) batch.find({'a': 2}).upsert().update_one({'$set': {'b': 2}}) batch.insert({'a': 3}) batch.find({'a': 3}).remove() result = batch.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 1, 'nInserted': 2, 'nRemoved': 1, 'upserted': [{'index': 2, '_id': '...'}], 'writeErrors': [], 'writeConcernErrors': []}, result) def test_single_error_unordered_batch(self): self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) batch = self.coll.initialize_unordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) batch.insert({'b': 3, 'a': 2}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 2, 'nRemoved': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [ {'index': 1, 'code': 11000, 'errmsg': '...', 'op': {'q': {'b': 2}, 'u': {'$set': {'a': 1}}, 'multi': False, 'upsert': True}}]}, result) def test_multiple_error_unordered_batch(self): self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) batch = self.coll.initialize_unordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 3}}) batch.find({'b': 3}).upsert().update_one({'$set': {'a': 4}}) batch.find({'b': 4}).upsert().update_one({'$set': {'a': 3}}) batch.insert({'b': 5, 'a': 2}) batch.insert({'b': 6, 'a': 1}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") # Assume the update at index 1 runs before the update at index 3, # although the spec does not require it. Same for inserts. self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 2, 'nInserted': 2, 'nRemoved': 0, 'upserted': [ {'index': 1, '_id': '...'}, {'index': 2, '_id': '...'}], 'writeConcernErrors': [], 'writeErrors': [ {'index': 3, 'code': 11000, 'errmsg': '...', 'op': {'q': {'b': 4}, 'u': {'$set': {'a': 3}}, 'multi': False, 'upsert': True}}, {'index': 5, 'code': 11000, 'errmsg': '...', 'op': {'_id': '...', 'b': 6, 'a': 1}}]}, result) def test_large_inserts_ordered(self): big = 'x' * self.coll.database.client.max_bson_size batch = self.coll.initialize_ordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.insert({'big': big}) batch.insert({'b': 2, 'a': 2}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(1, result['nInserted']) self.coll.delete_many({}) big = 'x' * (1024 * 1024 * 4) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1, 'big': big}) batch.insert({'a': 2, 'big': big}) batch.insert({'a': 3, 'big': big}) batch.insert({'a': 4, 'big': big}) batch.insert({'a': 5, 'big': big}) batch.insert({'a': 6, 'big': big}) result = batch.execute() self.assertEqual(6, result['nInserted']) self.assertEqual(6, self.coll.count()) def test_large_inserts_unordered(self): big = 'x' * self.coll.database.client.max_bson_size batch = self.coll.initialize_unordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.insert({'big': big}) batch.insert({'b': 2, 'a': 2}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(2, result['nInserted']) self.coll.delete_many({}) big = 'x' * (1024 * 1024 * 4) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1, 'big': big}) batch.insert({'a': 2, 'big': big}) batch.insert({'a': 3, 'big': big}) batch.insert({'a': 4, 'big': big}) batch.insert({'a': 5, 'big': big}) batch.insert({'a': 6, 'big': big}) result = batch.execute() self.assertEqual(6, result['nInserted']) self.assertEqual(6, self.coll.count()) def test_numerous_inserts(self): # Ensure we don't exceed server's 1000-document batch size limit. n_docs = 2100 batch = self.coll.initialize_unordered_bulk_op() for _ in range(n_docs): batch.insert({}) result = batch.execute() self.assertEqual(n_docs, result['nInserted']) self.assertEqual(n_docs, self.coll.count()) # Same with ordered bulk. self.coll.delete_many({}) batch = self.coll.initialize_ordered_bulk_op() for _ in range(n_docs): batch.insert({}) result = batch.execute() self.assertEqual(n_docs, result['nInserted']) self.assertEqual(n_docs, self.coll.count()) def test_multiple_execution(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({}) batch.execute() self.assertRaises(InvalidOperation, batch.execute) def test_generator_insert(self): def gen(): yield {'a': 1, 'b': 1} yield {'a': 1, 'b': 2} yield {'a': 2, 'b': 3} yield {'a': 3, 'b': 5} yield {'a': 5, 'b': 8} result = self.coll.insert_many(gen()) self.assertEqual(5, len(result.inserted_ids)) class TestLegacyBulkNoResults(BulkTestBase): @classmethod def setUpClass(cls): super(TestLegacyBulkNoResults, cls).setUpClass() cls.deprecation_filter = DeprecationFilter() @classmethod def tearDownClass(cls): cls.deprecation_filter.stop() def tearDown(self): self.coll.delete_many({}) def test_no_results_ordered_success(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'_id': 1}) batch.find({'_id': 3}).upsert().update_one({'$set': {'b': 1}}) batch.insert({'_id': 2}) batch.find({'_id': 1}).remove_one() self.assertTrue(batch.execute({'w': 0}) is None) wait_until(lambda: 2 == self.coll.count(), 'insert 2 documents') wait_until(lambda: self.coll.find_one({'_id': 1}) is None, 'removed {"_id": 1}') def test_no_results_ordered_failure(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'_id': 1}) batch.find({'_id': 3}).upsert().update_one({'$set': {'b': 1}}) batch.insert({'_id': 2}) # Fails with duplicate key error. batch.insert({'_id': 1}) # Should not be executed since the batch is ordered. batch.find({'_id': 1}).remove_one() self.assertTrue(batch.execute({'w': 0}) is None) wait_until(lambda: 3 == self.coll.count(), 'insert 3 documents') self.assertEqual({'_id': 1}, self.coll.find_one({'_id': 1})) def test_no_results_unordered_success(self): batch = self.coll.initialize_unordered_bulk_op() batch.insert({'_id': 1}) batch.find({'_id': 3}).upsert().update_one({'$set': {'b': 1}}) batch.insert({'_id': 2}) batch.find({'_id': 1}).remove_one() self.assertTrue(batch.execute({'w': 0}) is None) wait_until(lambda: 2 == self.coll.count(), 'insert 2 documents') wait_until(lambda: self.coll.find_one({'_id': 1}) is None, 'removed {"_id": 1}') def test_no_results_unordered_failure(self): batch = self.coll.initialize_unordered_bulk_op() batch.insert({'_id': 1}) batch.find({'_id': 3}).upsert().update_one({'$set': {'b': 1}}) batch.insert({'_id': 2}) # Fails with duplicate key error. batch.insert({'_id': 1}) # Should be executed since the batch is unordered. batch.find({'_id': 1}).remove_one() self.assertTrue(batch.execute({'w': 0}) is None) wait_until(lambda: 2 == self.coll.count(), 'insert 2 documents') wait_until(lambda: self.coll.find_one({'_id': 1}) is None, 'removed {"_id": 1}') class TestLegacyBulkWriteConcern(BulkTestBase): @classmethod def setUpClass(cls): super(TestLegacyBulkWriteConcern, cls).setUpClass() cls.w = client_context.w cls.secondary = None if cls.w > 1: for member in client_context.ismaster['hosts']: if member != client_context.ismaster['primary']: cls.secondary = single_client(*partition_node(member)) break # We tested wtimeout errors by specifying a write concern greater than # the number of members, but in MongoDB 2.7.8+ this causes a different # sort of error, "Not enough data-bearing nodes". In recent servers we # use a failpoint to pause replication on a secondary. cls.need_replication_stopped = client_context.version.at_least(2, 7, 8) cls.deprecation_filter = DeprecationFilter() @classmethod def tearDownClass(cls): cls.deprecation_filter.stop() def cause_wtimeout(self, batch): if self.need_replication_stopped: if not client_context.test_commands_enabled: raise SkipTest("Test commands must be enabled.") self.secondary.admin.command('configureFailPoint', 'rsSyncApplyStop', mode='alwaysOn') try: return batch.execute({'w': self.w, 'wtimeout': 1}) finally: self.secondary.admin.command('configureFailPoint', 'rsSyncApplyStop', mode='off') else: return batch.execute({'w': self.w + 1, 'wtimeout': 1}) def test_fsync_and_j(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) self.assertRaises( ConfigurationError, batch.execute, {'fsync': True, 'j': True}) @client_context.require_replica_set def test_write_concern_failure_ordered(self): # Ensure we don't raise on wnote. batch = self.coll.initialize_ordered_bulk_op() batch.find({"something": "that does no exist"}).remove() self.assertTrue(batch.execute({"w": self.w})) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) batch.insert({'a': 2}) # Replication wtimeout is a 'soft' error. # It shouldn't stop batch processing. try: self.cause_wtimeout(batch) except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 2, 'nRemoved': 0, 'upserted': [], 'writeErrors': []}, result) # When talking to legacy servers there will be a # write concern error for each operation. self.assertTrue(len(result['writeConcernErrors']) > 0) failed = result['writeConcernErrors'][0] self.assertEqual(64, failed['code']) self.assertTrue(isinstance(failed['errmsg'], string_type)) self.coll.delete_many({}) self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) # Fail due to write concern support as well # as duplicate key error on ordered batch. batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 3}).upsert().replace_one({'b': 1}) batch.insert({'a': 1}) batch.insert({'a': 2}) try: self.cause_wtimeout(batch) except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 1, 'nInserted': 1, 'nRemoved': 0, 'upserted': [{'index': 1, '_id': '...'}], 'writeErrors': [ {'index': 2, 'code': 11000, 'errmsg': '...', 'op': {'_id': '...', 'a': 1}}]}, result) self.assertTrue(len(result['writeConcernErrors']) > 1) failed = result['writeErrors'][0] self.assertTrue("duplicate" in failed['errmsg']) @client_context.require_replica_set def test_write_concern_failure_unordered(self): # Ensure we don't raise on wnote. batch = self.coll.initialize_unordered_bulk_op() batch.find({"something": "that does no exist"}).remove() self.assertTrue(batch.execute({"w": self.w})) batch = self.coll.initialize_unordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 3}).upsert().update_one({'$set': {'a': 3, 'b': 1}}) batch.insert({'a': 2}) # Replication wtimeout is a 'soft' error. # It shouldn't stop batch processing. try: self.cause_wtimeout(batch) except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(2, result['nInserted']) self.assertEqual(1, result['nUpserted']) self.assertEqual(0, len(result['writeErrors'])) # When talking to legacy servers there will be a # write concern error for each operation. self.assertTrue(len(result['writeConcernErrors']) > 1) self.coll.delete_many({}) self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) # Fail due to write concern support as well # as duplicate key error on unordered batch. batch = self.coll.initialize_unordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 3}).upsert().update_one({'$set': {'a': 3, 'b': 1}}) batch.insert({'a': 1}) batch.insert({'a': 2}) try: self.cause_wtimeout(batch) except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(2, result['nInserted']) self.assertEqual(1, result['nUpserted']) self.assertEqual(1, len(result['writeErrors'])) # When talking to legacy servers there will be a # write concern error for each operation. self.assertTrue(len(result['writeConcernErrors']) > 1) failed = result['writeErrors'][0] self.assertEqual(2, failed['index']) self.assertEqual(11000, failed['code']) self.assertTrue(isinstance(failed['errmsg'], string_type)) self.assertEqual(1, failed['op']['a']) failed = result['writeConcernErrors'][0] self.assertEqual(64, failed['code']) self.assertTrue(isinstance(failed['errmsg'], string_type)) upserts = result['upserted'] self.assertEqual(1, len(upserts)) self.assertEqual(1, upserts[0]['index']) self.assertTrue(upserts[0].get('_id')) class TestLegacyBulkAuthorization(BulkAuthorizationTestBase): @classmethod def setUpClass(cls): super(TestLegacyBulkAuthorization, cls).setUpClass() cls.deprecation_filter = DeprecationFilter() @classmethod def tearDownClass(cls): cls.deprecation_filter.stop() def test_readonly(self): # We test that an authorization failure aborts the batch and is raised # as OperationFailure. cli = rs_or_single_client_noauth() db = cli.pymongo_test coll = db.test db.authenticate('readonly', 'pw') bulk = coll.initialize_ordered_bulk_op() bulk.insert({'x': 1}) self.assertRaises(OperationFailure, bulk.execute) def test_no_remove(self): # We test that an authorization failure aborts the batch and is raised # as OperationFailure. cli = rs_or_single_client_noauth() db = cli.pymongo_test coll = db.test db.authenticate('noremove', 'pw') bulk = coll.initialize_ordered_bulk_op() bulk.insert({'x': 1}) bulk.find({'x': 2}).upsert().replace_one({'x': 2}) bulk.find({}).remove() # Prohibited. bulk.insert({'x': 3}) # Never attempted. self.assertRaises(OperationFailure, bulk.execute) self.assertEqual(set([1, 2]), set(self.coll.distinct('x'))) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_monotonic.py0000644000076600000240000000233113245621354020034 0ustar shanestaff00000000000000# Copyright 2018-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the monotonic module.""" import sys sys.path[0:0] = [""] from pymongo.monotonic import time as pymongo_time from test import unittest class TestMonotonic(unittest.TestCase): def test_monotonic_time(self): try: from monotonic import monotonic self.assertIs(monotonic, pymongo_time) except ImportError: if sys.version_info[:2] >= (3, 3): from time import monotonic self.assertIs(monotonic, pymongo_time) else: from time import time self.assertIs(time, pymongo_time) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_gridfs_spec.py0000644000076600000240000001772113245617773020342 0ustar shanestaff00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test GridFSBucket class.""" import copy import datetime import os import sys import re import gridfs sys.path[0:0] = [""] from bson import Binary from bson.json_util import loads from bson.py3compat import bytes_from_hex from gridfs.errors import NoFile, CorruptGridFile from test import (unittest, IntegrationTest) # Commands. _COMMANDS = {"delete": lambda coll, doc: [coll.delete_many(d["q"]) for d in doc['deletes']], "insert": lambda coll, doc: coll.insert_many(doc['documents']), "update": lambda coll, doc: [coll.update_many(u["q"], u["u"]) for u in doc['updates']] } # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'gridfs') def camel_to_snake(camel): # Regex to convert CamelCase to snake_case. Special case for _id. if camel == "id": return "file_id" snake = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', camel) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', snake).lower() class TestAllScenarios(IntegrationTest): @classmethod def setUpClass(cls): super(TestAllScenarios, cls).setUpClass() cls.fs = gridfs.GridFSBucket(cls.db) cls.str_to_cmd = { "upload": cls.fs.upload_from_stream, "download": cls.fs.open_download_stream, "delete": cls.fs.delete, "download_by_name": cls.fs.open_download_stream_by_name} def init_db(self, data, test): self.db.drop_collection("fs.files") self.db.drop_collection("fs.chunks") self.db.drop_collection("expected.files") self.db.drop_collection("expected.chunks") # Read in data. if data['files']: self.db.fs.files.insert_many(data['files']) self.db.expected.files.insert_many(data['files']) if data['chunks']: self.db.fs.chunks.insert_many(data['chunks']) self.db.expected.chunks.insert_many(data['chunks']) # Make initial modifications. if "arrange" in test: for cmd in test['arrange'].get('data', []): for key in cmd.keys(): if key in _COMMANDS: coll = self.db.get_collection(cmd[key]) _COMMANDS[key](coll, cmd) def init_expected_db(self, test, result): # Modify outcome DB. for cmd in test['assert'].get('data', []): for key in cmd.keys(): if key in _COMMANDS: # Replace wildcards in inserts. for doc in cmd.get('documents', []): keylist = doc.keys() for dockey in copy.deepcopy(list(keylist)): if "result" in str(doc[dockey]): doc[dockey] = result if "actual" in str(doc[dockey]): # Avoid duplicate doc.pop(dockey) # Move contentType to metadata. if dockey == "contentType": doc["metadata"] = {dockey: doc.pop(dockey)} coll = self.db.get_collection(cmd[key]) _COMMANDS[key](coll, cmd) if test['assert'].get('result') == "&result": test['assert']['result'] = result def sorted_list(self, coll, ignore_id): to_sort = [] for doc in coll.find(): docstr = "{" if ignore_id: # Cannot compare _id in chunks collection. doc.pop("_id") for k in sorted(doc.keys()): if k == "uploadDate": # Can't compare datetime. self.assertTrue(isinstance(doc[k], datetime.datetime)) else: docstr += "%s:%s " % (k, repr(doc[k])) to_sort.append(docstr + "}") return to_sort def create_test(scenario_def): def run_scenario(self): # Run tests. self.assertTrue(scenario_def['tests'], "tests cannot be empty") for test in scenario_def['tests']: self.init_db(scenario_def['data'], test) # Run GridFs Operation. operation = self.str_to_cmd[test['act']['operation']] args = test['act']['arguments'] extra_opts = args.pop("options", {}) if "contentType" in extra_opts: extra_opts["metadata"] = { "contentType": extra_opts.pop("contentType")} args.update(extra_opts) converted_args = dict((camel_to_snake(c), v) for c, v in args.items()) error = None try: result = operation(**converted_args) if 'download' in test['act']['operation']: result = Binary(result.read()) except Exception as exc: error = exc self.init_expected_db(test, result) # Asserts. errors = {"FileNotFound": NoFile, "ChunkIsMissing": CorruptGridFile, "ExtraChunk": CorruptGridFile, "ChunkIsWrongSize": CorruptGridFile, "RevisionNotFound": NoFile} if test['assert'].get("error", False): self.assertIsNotNone(error) self.assertTrue(isinstance(error, errors[test['assert']['error']])) else: self.assertIsNone(error) if 'result' in test['assert']: if test['assert']['result'] == 'void': test['assert']['result'] = None self.assertEqual(result, test['assert'].get('result')) if 'data' in test['assert']: # Create alphabetized list self.assertEqual( set(self.sorted_list(self.db.fs.chunks, True)), set(self.sorted_list(self.db.expected.chunks, True))) self.assertEqual( set(self.sorted_list(self.db.fs.files, False)), set(self.sorted_list(self.db.expected.files, False))) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = loads(scenario_stream.read()) # Because object_hook is already defined by bson.json_util, # and everything is named 'data' def str2hex(jsn): for key, val in jsn.items(): if key in ("data", "source", "result"): if "$hex" in val: jsn[key] = Binary(bytes_from_hex(val['$hex'])) if isinstance(jsn[key], dict): str2hex(jsn[key]) if isinstance(jsn[key], list): for k in jsn[key]: str2hex(k) str2hex(scenario_def) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s' % ( os.path.splitext(filename)[0]) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_threads.py0000644000076600000240000001424513245621354017470 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test that pymongo is thread safe.""" import threading from test import (client_context, db_user, db_pwd, IntegrationTest, unittest) from test.utils import rs_or_single_client_noauth, rs_or_single_client from test.utils import joinall from pymongo.errors import OperationFailure @client_context.require_connection def setUpModule(): pass class AutoAuthenticateThreads(threading.Thread): def __init__(self, collection, num): threading.Thread.__init__(self) self.coll = collection self.num = num self.success = False self.setDaemon(True) def run(self): for i in range(self.num): self.coll.insert_one({'num': i}) self.coll.find_one({'num': i}) self.success = True class SaveAndFind(threading.Thread): def __init__(self, collection): threading.Thread.__init__(self) self.collection = collection self.setDaemon(True) self.passed = False def run(self): sum = 0 for document in self.collection.find(): sum += document["x"] assert sum == 499500, "sum was %d not 499500" % sum self.passed = True class Insert(threading.Thread): def __init__(self, collection, n, expect_exception): threading.Thread.__init__(self) self.collection = collection self.n = n self.expect_exception = expect_exception self.setDaemon(True) def run(self): for _ in range(self.n): error = True try: self.collection.insert_one({"test": "insert"}) error = False except: if not self.expect_exception: raise if self.expect_exception: assert error class Update(threading.Thread): def __init__(self, collection, n, expect_exception): threading.Thread.__init__(self) self.collection = collection self.n = n self.expect_exception = expect_exception self.setDaemon(True) def run(self): for _ in range(self.n): error = True try: self.collection.update_one({"test": "unique"}, {"$set": {"test": "update"}}) error = False except: if not self.expect_exception: raise if self.expect_exception: assert error class Disconnect(threading.Thread): def __init__(self, client, n): threading.Thread.__init__(self) self.client = client self.n = n self.passed = False def run(self): for _ in range(self.n): self.client.close() self.passed = True class TestThreads(IntegrationTest): def setUp(self): self.db = self.client.pymongo_test def test_threading(self): self.db.drop_collection("test") self.db.test.insert_many([{"x": i} for i in range(1000)]) threads = [] for i in range(10): t = SaveAndFind(self.db.test) t.start() threads.append(t) joinall(threads) def test_safe_insert(self): self.db.drop_collection("test1") self.db.test1.insert_one({"test": "insert"}) self.db.drop_collection("test2") self.db.test2.insert_one({"test": "insert"}) self.db.test2.create_index("test", unique=True) self.db.test2.find_one() okay = Insert(self.db.test1, 2000, False) error = Insert(self.db.test2, 2000, True) error.start() okay.start() error.join() okay.join() def test_safe_update(self): self.db.drop_collection("test1") self.db.test1.insert_one({"test": "update"}) self.db.test1.insert_one({"test": "unique"}) self.db.drop_collection("test2") self.db.test2.insert_one({"test": "update"}) self.db.test2.insert_one({"test": "unique"}) self.db.test2.create_index("test", unique=True) self.db.test2.find_one() okay = Update(self.db.test1, 2000, False) error = Update(self.db.test2, 2000, True) error.start() okay.start() error.join() okay.join() def test_client_disconnect(self): db = rs_or_single_client(serverSelectionTimeoutMS=30000).pymongo_test db.drop_collection("test") db.test.insert_many([{"x": i} for i in range(1000)]) # Start 10 threads that execute a query, and 10 threads that call # client.close() 10 times in a row. threads = [SaveAndFind(db.test) for _ in range(10)] threads.extend(Disconnect(db.client, 10) for _ in range(10)) for t in threads: t.start() for t in threads: t.join(300) for t in threads: self.assertTrue(t.passed) class TestThreadsAuth(IntegrationTest): @classmethod @client_context.require_auth def setUpClass(cls): super(TestThreadsAuth, cls).setUpClass() def test_auto_auth_login(self): client = rs_or_single_client_noauth() self.assertRaises(OperationFailure, client.auth_test.test.find_one) # Admin auth client.admin.authenticate(db_user, db_pwd) nthreads = 10 threads = [] for _ in range(nthreads): t = AutoAuthenticateThreads(client.auth_test.test, 10) t.start() threads.append(t) joinall(threads) for t in threads: self.assertTrue(t.success) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/utils_selection_tests.py0000644000076600000240000002350413245621354021424 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for testing Server Selection and Max Staleness.""" import datetime import os import sys sys.path[0:0] = [""] from bson import json_util from pymongo import read_preferences from pymongo.common import clean_node, HEARTBEAT_FREQUENCY from pymongo.errors import AutoReconnect, ConfigurationError from pymongo.ismaster import IsMaster from pymongo.server_description import ServerDescription from pymongo.settings import TopologySettings from pymongo.server_selectors import writable_server_selector from pymongo.topology import Topology from test import unittest class MockSocketInfo(object): def close(self): pass def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass class MockPool(object): def __init__(self, *args, **kwargs): pass def reset(self): pass def remove_stale_sockets(self): pass class MockMonitor(object): def __init__(self, server_description, topology, pool, topology_settings): pass def open(self): pass def request_check(self): pass def close(self): pass def get_addresses(server_list): seeds = [] hosts = [] for server in server_list: seeds.append(clean_node(server['address'])) hosts.append(server['address']) return seeds, hosts def make_last_write_date(server): epoch = datetime.datetime.utcfromtimestamp(0) millis = server.get('lastWrite', {}).get('lastWriteDate') if millis: diff = ((millis % 1000) + 1000) % 1000 seconds = (millis - diff) / 1000 micros = diff * 1000 return epoch + datetime.timedelta( seconds=seconds, microseconds=micros) else: # "Unknown" server. return epoch def make_server_description(server, hosts): """Make a ServerDescription from server info in a JSON test.""" server_type = server['type'] if server_type == "Unknown": return ServerDescription(clean_node(server['address']), IsMaster({})) ismaster_response = {'ok': True, 'hosts': hosts} if server_type != "Standalone" and server_type != "Mongos": ismaster_response['setName'] = "rs" if server_type == "RSPrimary": ismaster_response['ismaster'] = True elif server_type == "RSSecondary": ismaster_response['secondary'] = True elif server_type == "Mongos": ismaster_response['msg'] = 'isdbgrid' ismaster_response['lastWrite'] = { 'lastWriteDate': make_last_write_date(server) } for field in 'maxWireVersion', 'tags', 'idleWritePeriodMillis': if field in server: ismaster_response[field] = server[field] ismaster_response.setdefault('maxWireVersion', 6) # Sets _last_update_time to now. sd = ServerDescription(clean_node(server['address']), IsMaster(ismaster_response), round_trip_time=server['avg_rtt_ms'] / 1000.0) if 'lastUpdateTime' in server: sd._last_update_time = server['lastUpdateTime'] / 1000.0 # ms to sec. return sd def get_topology_type_name(scenario_def): td = scenario_def['topology_description'] name = td['type'] if name == 'Unknown': # PyMongo never starts a topology in type Unknown. return 'Sharded' if len(td['servers']) > 1 else 'Single' else: return name def create_test(scenario_def): def run_scenario(self): # Initialize topologies. if 'heartbeatFrequencyMS' in scenario_def: frequency = int(scenario_def['heartbeatFrequencyMS']) / 1000.0 else: frequency = HEARTBEAT_FREQUENCY settings = dict( monitor_class=MockMonitor, heartbeat_frequency=frequency, pool_class=MockPool) settings['seeds'], hosts = get_addresses( scenario_def['topology_description']['servers']) # "Eligible servers" is defined in the server selection spec as # the set of servers matching both the ReadPreference's mode # and tag sets. top_latency = Topology(TopologySettings(**settings)) top_latency.open() # "In latency window" is defined in the server selection # spec as the subset of suitable_servers that falls within the # allowable latency window. settings['local_threshold_ms'] = 1000000 top_suitable = Topology(TopologySettings(**settings)) top_suitable.open() # Update topologies with server descriptions. for server in scenario_def['topology_description']['servers']: server_description = make_server_description(server, hosts) top_suitable.on_change(server_description) top_latency.on_change(server_description) # Create server selector. if scenario_def.get("operation") == "write": pref = writable_server_selector else: # Make first letter lowercase to match read_pref's modes. pref_def = scenario_def['read_preference'] mode_string = pref_def.get('mode', 'primary') mode_string = mode_string[:1].lower() + mode_string[1:] mode = read_preferences.read_pref_mode_from_name(mode_string) max_staleness = pref_def.get('maxStalenessSeconds', -1) tag_sets = pref_def.get('tag_sets') if scenario_def.get('error'): with self.assertRaises((ConfigurationError, ValueError)): # Error can be raised when making Read Pref or selecting. pref = read_preferences.make_read_preference( mode, tag_sets=tag_sets, max_staleness=max_staleness) top_latency.select_server(pref) return pref = read_preferences.make_read_preference( mode, tag_sets=tag_sets, max_staleness=max_staleness) # Select servers. if not scenario_def.get('suitable_servers'): with self.assertRaises(AutoReconnect): top_suitable.select_server(pref, server_selection_timeout=0) return if not scenario_def['in_latency_window']: with self.assertRaises(AutoReconnect): top_latency.select_server(pref, server_selection_timeout=0) return actual_suitable_s = top_suitable.select_servers( pref, server_selection_timeout=0) actual_latency_s = top_latency.select_servers( pref, server_selection_timeout=0) expected_suitable_servers = {} for server in scenario_def['suitable_servers']: server_description = make_server_description(server, hosts) expected_suitable_servers[server['address']] = server_description actual_suitable_servers = {} for s in actual_suitable_s: actual_suitable_servers["%s:%d" % (s.description.address[0], s.description.address[1])] = s.description self.assertEqual(len(actual_suitable_servers), len(expected_suitable_servers)) for k, actual in actual_suitable_servers.items(): expected = expected_suitable_servers[k] self.assertEqual(expected.address, actual.address) self.assertEqual(expected.server_type, actual.server_type) self.assertEqual(expected.round_trip_time, actual.round_trip_time) self.assertEqual(expected.tags, actual.tags) self.assertEqual(expected.all_hosts, actual.all_hosts) expected_latency_servers = {} for server in scenario_def['in_latency_window']: server_description = make_server_description(server, hosts) expected_latency_servers[server['address']] = server_description actual_latency_servers = {} for s in actual_latency_s: actual_latency_servers["%s:%d" % (s.description.address[0], s.description.address[1])] = s.description self.assertEqual(len(actual_latency_servers), len(expected_latency_servers)) for k, actual in actual_latency_servers.items(): expected = expected_latency_servers[k] self.assertEqual(expected.address, actual.address) self.assertEqual(expected.server_type, actual.server_type) self.assertEqual(expected.round_trip_time, actual.round_trip_time) self.assertEqual(expected.tags, actual.tags) self.assertEqual(expected.all_hosts, actual.all_hosts) return run_scenario def create_selection_tests(test_dir): class TestAllScenarios(unittest.TestCase): pass for dirpath, _, filenames in os.walk(test_dir): dirname = os.path.split(dirpath) dirname = os.path.split(dirname[-2])[-1] + '_' + dirname[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json_util.loads(scenario_stream.read()) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s_%s' % ( dirname, os.path.splitext(filename)[0]) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) return TestAllScenarios pymongo-3.6.1/test/test_discovery_and_monitoring.py0000644000076600000240000001654713245621354023143 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the topology module.""" import os import sys import threading sys.path[0:0] = [""] from bson import json_util, Timestamp from pymongo import common from pymongo.errors import ConfigurationError from pymongo.topology import Topology from pymongo.topology_description import TOPOLOGY_TYPE from pymongo.ismaster import IsMaster from pymongo.server_description import ServerDescription, SERVER_TYPE from pymongo.settings import TopologySettings from pymongo.uri_parser import parse_uri from test import unittest # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'discovery_and_monitoring') class MockSocketInfo(object): def close(self): pass def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass class MockPool(object): def __init__(self, *args, **kwargs): self.pool_id = 0 self._lock = threading.Lock() def reset(self): with self._lock: self.pool_id += 1 class MockMonitor(object): def __init__(self, server_description, topology, pool, topology_settings): self._server_description = server_description self._topology = topology def open(self): pass def request_check(self): pass def close(self): pass def remove_stale_sockets(self): pass def create_mock_topology(uri, monitor_class=MockMonitor): # Some tests in the spec include URIs like mongodb://A/?connect=direct, # but PyMongo considers any single-seed URI with no setName to be "direct". parsed_uri = parse_uri(uri.replace('connect=direct', '')) replica_set_name = None if 'replicaset' in parsed_uri['options']: replica_set_name = parsed_uri['options']['replicaset'] topology_settings = TopologySettings( parsed_uri['nodelist'], replica_set_name=replica_set_name, pool_class=MockPool, monitor_class=monitor_class) c = Topology(topology_settings) c.open() return c def got_ismaster(topology, server_address, ismaster_response): server_description = ServerDescription( server_address, IsMaster(ismaster_response), 0) topology.on_change(server_description) def get_type(topology, hostname): description = topology.get_server_by_address((hostname, 27017)).description return description.server_type class TestAllScenarios(unittest.TestCase): pass def topology_type_name(topology_type): return TOPOLOGY_TYPE._fields[topology_type] def server_type_name(server_type): return SERVER_TYPE._fields[server_type] def check_outcome(self, topology, outcome): expected_servers = outcome['servers'] # Check weak equality before proceeding. self.assertEqual( len(topology.description.server_descriptions()), len(expected_servers)) if outcome.get('compatible') is False: with self.assertRaises(ConfigurationError): topology.description.check_compatible() else: # No error. topology.description.check_compatible() # Since lengths are equal, every actual server must have a corresponding # expected server. for expected_server_address, expected_server in expected_servers.items(): node = common.partition_node(expected_server_address) self.assertTrue(topology.has_server(node)) actual_server = topology.get_server_by_address(node) actual_server_description = actual_server.description if expected_server['type'] == 'PossiblePrimary': # Special case, some tests in the spec include the PossiblePrimary # type, but only single-threaded drivers need that type. We call # possible primaries Unknown. expected_server_type = SERVER_TYPE.Unknown else: expected_server_type = getattr( SERVER_TYPE, expected_server['type']) self.assertEqual( server_type_name(expected_server_type), server_type_name(actual_server_description.server_type)) self.assertEqual( expected_server.get('setName'), actual_server_description.replica_set_name) self.assertEqual( expected_server.get('setVersion'), actual_server_description.set_version) self.assertEqual( expected_server.get('electionId'), actual_server_description.election_id) self.assertEqual(outcome['setName'], topology.description.replica_set_name) self.assertEqual(outcome['logicalSessionTimeoutMinutes'], topology.description.logical_session_timeout_minutes) expected_topology_type = getattr(TOPOLOGY_TYPE, outcome['topologyType']) self.assertEqual(topology_type_name(expected_topology_type), topology_type_name(topology.description.topology_type)) def create_test(scenario_def): def run_scenario(self): c = create_mock_topology(scenario_def['uri']) for phase in scenario_def['phases']: for response in phase['responses']: got_ismaster(c, common.partition_node(response[0]), response[1]) check_outcome(self, c, phase['outcome']) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): dirname = os.path.split(dirpath)[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json_util.loads(scenario_stream.read()) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s_%s' % ( dirname, os.path.splitext(filename)[0]) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() class TestClusterTimeComparison(unittest.TestCase): def test_cluster_time_comparison(self): t = create_mock_topology('mongodb://host') def send_cluster_time(time, inc, should_update): old = t.max_cluster_time() new = {'clusterTime': Timestamp(time, inc)} got_ismaster(t, ('host', 27017), {'ok': 1, 'minWireVersion': 0, 'maxWireVersion': 6, '$clusterTime': new}) actual = t.max_cluster_time() if should_update: self.assertEqual(actual, new) else: self.assertEqual(actual, old) send_cluster_time(0, 1, True) send_cluster_time(2, 2, True) send_cluster_time(2, 1, False) send_cluster_time(1, 3, False) send_cluster_time(2, 3, True) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_crud.py0000644000076600000240000002472613245621354017000 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the collection module.""" import json import os import re import sys sys.path[0:0] = [""] from bson.py3compat import iteritems from pymongo import operations from pymongo.command_cursor import CommandCursor from pymongo.cursor import Cursor from pymongo.results import _WriteResult, BulkWriteResult from pymongo.operations import (InsertOne, DeleteOne, DeleteMany, ReplaceOne, UpdateOne, UpdateMany) from test import unittest, client_context, IntegrationTest # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'crud') class TestAllScenarios(IntegrationTest): pass def camel_to_snake(camel): # Regex to convert CamelCase to snake_case. snake = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', camel) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', snake).lower() def camel_to_upper_camel(camel): return camel[0].upper() + camel[1:] def check_result(expected_result, result): if isinstance(result, Cursor) or isinstance(result, CommandCursor): return list(result) == expected_result elif isinstance(result, _WriteResult): for res in expected_result: prop = camel_to_snake(res) # SPEC-869: Only BulkWriteResult has upserted_count. if (prop == "upserted_count" and not isinstance(result, BulkWriteResult)): if result.upserted_id is not None: upserted_count = 1 else: upserted_count = 0 if upserted_count != expected_result[res]: return False elif prop == "inserted_ids": # BulkWriteResult does not have inserted_ids. if isinstance(result, BulkWriteResult): if len(expected_result[res]) != result.inserted_count: return False else: # InsertManyResult may be compared to [id1] from the # crud spec or {"0": id1} from the retryable write spec. ids = expected_result[res] if isinstance(ids, dict): ids = [ids[str(i)] for i in range(len(ids))] if ids != result.inserted_ids: return False elif prop == "upserted_ids": # Convert indexes from strings to integers. ids = expected_result[res] expected_ids = {} for str_index in ids: expected_ids[int(str_index)] = ids[str_index] if expected_ids != result.upserted_ids: return False elif getattr(result, prop) != expected_result[res]: return False return True else: if not expected_result: return result is None else: return result == expected_result def camel_to_snake_args(arguments): for arg_name in list(arguments): c2s = camel_to_snake(arg_name) arguments[c2s] = arguments.pop(arg_name) return arguments def run_operation(collection, test): # Convert command from CamelCase to pymongo.collection method. operation = camel_to_snake(test['operation']['name']) cmd = getattr(collection, operation) # Convert arguments to snake_case and handle special cases. arguments = test['operation']['arguments'] options = arguments.pop("options", {}) for option_name in options: arguments[camel_to_snake(option_name)] = options[option_name] if operation == "bulk_write": # Parse each request into a bulk write model. requests = [] for request in arguments["requests"]: bulk_model = camel_to_upper_camel(request["name"]) bulk_class = getattr(operations, bulk_model) bulk_arguments = camel_to_snake_args(request["arguments"]) requests.append(bulk_class(**bulk_arguments)) arguments["requests"] = requests else: for arg_name in list(arguments): c2s = camel_to_snake(arg_name) # PyMongo accepts sort as list of tuples. Asserting len=1 # because ordering dicts from JSON in 2.6 is unwieldy. if arg_name == "sort": sort_dict = arguments[arg_name] assert len(sort_dict) == 1, 'test can only have 1 sort key' arguments[arg_name] = list(iteritems(sort_dict)) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) # Aggregate uses "batchSize", while find uses batch_size. elif arg_name == "batchSize" and operation == "aggregate": continue # Requires boolean returnDocument. elif arg_name == "returnDocument": arguments[c2s] = arguments[arg_name] == "After" else: arguments[c2s] = arguments.pop(arg_name) result = cmd(**arguments) if operation == "aggregate": if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]: out = collection.database[arguments["pipeline"][-1]["$out"]] return out.find() return result def create_test(scenario_def, test): def run_scenario(self): # Load data. assert scenario_def['data'], "tests must have non-empty data" for name in self.db.collection_names(False): self.db.drop_collection(name) self.db.test.insert_many(scenario_def['data']) result = run_operation(self.db.test, test) # Assert final state is expected. expected_c = test['outcome'].get('collection') if expected_c is not None: expected_name = expected_c.get('name') if expected_name is not None: db_coll = self.db[expected_name] else: db_coll = self.db.test self.assertEqual(list(db_coll.find()), expected_c['data']) expected_result = test['outcome'].get('result') self.assertTrue(check_result(expected_result, result)) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): dirname = os.path.split(dirpath)[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json.load(scenario_stream) test_type = os.path.splitext(filename)[0] min_ver, max_ver = None, None if 'minServerVersion' in scenario_def: min_ver = tuple( int(elt) for elt in scenario_def['minServerVersion'].split('.')) if 'maxServerVersion' in scenario_def: max_ver = tuple( int(elt) for elt in scenario_def['maxServerVersion'].split('.')) # Construct test from scenario. for test in scenario_def['tests']: new_test = create_test(scenario_def, test) if min_ver is not None: new_test = client_context.require_version_min(*min_ver)( new_test) if max_ver is not None: new_test = client_context.require_version_max(*max_ver)( new_test) test_name = 'test_%s_%s_%s' % ( dirname, test_type, str(test['description'].replace(" ", "_"))) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() class TestWriteOpsComparison(unittest.TestCase): def test_InsertOneEquals(self): self.assertEqual(InsertOne({'foo': 42}), InsertOne({'foo': 42})) def test_InsertOneNotEquals(self): self.assertNotEqual(InsertOne({'foo': 42}), InsertOne({'foo': 23})) def test_DeleteOneEquals(self): self.assertEqual(DeleteOne({'foo': 42}), DeleteOne({'foo': 42})) def test_DeleteOneNotEquals(self): self.assertNotEqual(DeleteOne({'foo': 42}), DeleteOne({'foo': 23})) def test_DeleteManyEquals(self): self.assertEqual(DeleteMany({'foo': 42}), DeleteMany({'foo': 42})) def test_DeleteManyNotEquals(self): self.assertNotEqual(DeleteMany({'foo': 42}), DeleteMany({'foo': 23})) def test_DeleteOneNotEqualsDeleteMany(self): self.assertNotEqual(DeleteOne({'foo': 42}), DeleteMany({'foo': 42})) def test_ReplaceOneEquals(self): self.assertEqual(ReplaceOne({'foo': 42}, {'bar': 42}, upsert=False), ReplaceOne({'foo': 42}, {'bar': 42}, upsert=False)) def test_ReplaceOneNotEquals(self): self.assertNotEqual(ReplaceOne({'foo': 42}, {'bar': 42}, upsert=False), ReplaceOne({'foo': 42}, {'bar': 42}, upsert=True)) def test_UpdateOneEquals(self): self.assertEqual(UpdateOne({'foo': 42}, {'$set': {'bar': 42}}), UpdateOne({'foo': 42}, {'$set': {'bar': 42}})) def test_UpdateOneNotEquals(self): self.assertNotEqual(UpdateOne({'foo': 42}, {'$set': {'bar': 42}}), UpdateOne({'foo': 42}, {'$set': {'bar': 23}})) def test_UpdateManyEquals(self): self.assertEqual(UpdateMany({'foo': 42}, {'$set': {'bar': 42}}), UpdateMany({'foo': 42}, {'$set': {'bar': 42}})) def test_UpdateManyNotEquals(self): self.assertNotEqual(UpdateMany({'foo': 42}, {'$set': {'bar': 42}}), UpdateMany({'foo': 42}, {'$set': {'bar': 23}})) def test_UpdateOneNotEqualsUpdateMany(self): self.assertNotEqual(UpdateOne({'foo': 42}, {'$set': {'bar': 42}}), UpdateMany({'foo': 42}, {'$set': {'bar': 42}})) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_replica_set_reconfig.py0000644000076600000240000001350413245621354022201 0ustar shanestaff00000000000000# Copyright 2013-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test clients and replica set configuration changes, using mocks.""" import sys sys.path[0:0] = [""] from pymongo.errors import ConnectionFailure, AutoReconnect from pymongo import ReadPreference from test import unittest, client_context, client_knobs, MockClientTest from test.pymongo_mocks import MockClient from test.utils import wait_until @client_context.require_connection def setUpModule(): pass class TestSecondaryBecomesStandalone(MockClientTest): # An administrator removes a secondary from a 3-node set and # brings it back up as standalone, without updating the other # members' config. Verify we don't continue using it. def test_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2', 'c:3'], mongoses=[], host='a:1,b:2,c:3', replicaSet='rs', serverSelectionTimeoutMS=100) self.addCleanup(c.close) # MongoClient connects to primary by default. wait_until(lambda: c.address is not None, 'connect to primary') self.assertEqual(c.address, ('a', 1)) # C is brought up as a standalone. c.mock_members.remove('c:3') c.mock_standalones.append('c:3') # Fail over. c.kill_host('a:1') c.kill_host('b:2') # Force reconnect. c.close() with self.assertRaises(AutoReconnect): c.db.command('ismaster') self.assertEqual(c.address, None) def test_replica_set_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2', 'c:3'], mongoses=[], host='a:1,b:2,c:3', replicaSet='rs') self.addCleanup(c.close) wait_until(lambda: ('b', 2) in c.secondaries, 'discover host "b"') wait_until(lambda: ('c', 3) in c.secondaries, 'discover host "c"') # C is brought up as a standalone. c.mock_members.remove('c:3') c.mock_standalones.append('c:3') wait_until(lambda: set([('b', 2)]) == c.secondaries, 'update the list of secondaries') self.assertEqual(('a', 1), c.primary) class TestSecondaryRemoved(MockClientTest): # An administrator removes a secondary from a 3-node set *without* # restarting it as standalone. def test_replica_set_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2', 'c:3'], mongoses=[], host='a:1,b:2,c:3', replicaSet='rs') self.addCleanup(c.close) wait_until(lambda: ('b', 2) in c.secondaries, 'discover host "b"') wait_until(lambda: ('c', 3) in c.secondaries, 'discover host "c"') # C is removed. c.mock_ismaster_hosts.remove('c:3') wait_until(lambda: set([('b', 2)]) == c.secondaries, 'update list of secondaries') self.assertEqual(('a', 1), c.primary) class TestSocketError(MockClientTest): def test_socket_error_marks_member_down(self): # Disable background refresh. with client_knobs(heartbeat_frequency=999999): c = MockClient( standalones=[], members=['a:1', 'b:2'], mongoses=[], host='a:1', replicaSet='rs') self.addCleanup(c.close) wait_until(lambda: len(c.nodes) == 2, 'discover both nodes') # b now raises socket.error. c.mock_down_hosts.append('b:2') self.assertRaises( ConnectionFailure, c.db.collection.with_options( read_preference=ReadPreference.SECONDARY).find_one) self.assertEqual(1, len(c.nodes)) class TestSecondaryAdded(MockClientTest): def test_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2'], mongoses=[], host='a:1', replicaSet='rs') self.addCleanup(c.close) wait_until(lambda: len(c.nodes) == 2, 'discover both nodes') # MongoClient connects to primary by default. self.assertEqual(c.address, ('a', 1)) self.assertEqual(set([('a', 1), ('b', 2)]), c.nodes) # C is added. c.mock_members.append('c:3') c.mock_ismaster_hosts.append('c:3') c.close() c.db.command('ismaster') self.assertEqual(c.address, ('a', 1)) wait_until(lambda: set([('a', 1), ('b', 2), ('c', 3)]) == c.nodes, 'reconnect to both secondaries') def test_replica_set_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2'], mongoses=[], host='a:1', replicaSet='rs') self.addCleanup(c.close) wait_until(lambda: ('a', 1) == c.primary, 'discover the primary') wait_until(lambda: set([('b', 2)]) == c.secondaries, 'discover the secondary') # C is added. c.mock_members.append('c:3') c.mock_ismaster_hosts.append('c:3') wait_until(lambda: set([('b', 2), ('c', 3)]) == c.secondaries, 'discover the new secondary') self.assertEqual(('a', 1), c.primary) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_sdam_monitoring_spec.py0000644000076600000240000002722213245617773022252 0ustar shanestaff00000000000000# Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Run the sdam monitoring spec tests.""" import json import os import sys import weakref sys.path[0:0] = [""] from bson.json_util import object_hook from pymongo import monitoring from pymongo import periodic_executor from pymongo.ismaster import IsMaster from pymongo.monitor import Monitor from pymongo.read_preferences import MovingAverage from pymongo.server_description import ServerDescription from pymongo.server_type import SERVER_TYPE from pymongo.topology import TOPOLOGY_TYPE from test import unittest, client_context, client_knobs from test.utils import (ServerAndTopologyEventListener, single_client, wait_until) # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'sdam_monitoring') def compare_server_descriptions(expected, actual): if ((not expected['address'] == "%s:%s" % actual.address) or (not SERVER_TYPE.__getattribute__(expected['type']) == actual.server_type)): return False expected_hosts = set( expected['arbiters'] + expected['passives'] + expected['hosts']) return expected_hosts == set("%s:%s" % s for s in actual.all_hosts) def compare_topology_descriptions(expected, actual): if not (TOPOLOGY_TYPE.__getattribute__( expected['topologyType']) == actual.topology_type): return False expected = expected['servers'] actual = actual.server_descriptions() if len(expected) != len(actual): return False for exp_server in expected: for address, actual_server in actual.items(): if compare_server_descriptions(exp_server, actual_server): break else: return False return True def compare_events(expected_dict, actual): if not expected_dict: return False, "Error: Bad expected value in YAML test" if not actual: return False, "Error: Event published was None" expected_type, expected = list(expected_dict.items())[0] if expected_type == "server_opening_event": if not isinstance(actual, monitoring.ServerOpeningEvent): return False, "Expected ServerOpeningEvent, got %s" % ( actual.__class__) if not expected['address'] == "%s:%s" % actual.server_address: return (False, "ServerOpeningEvent published with wrong address (expected" " %s, got %s" % (expected['address'], actual.server_address)) elif expected_type == "server_description_changed_event": if not isinstance(actual, monitoring.ServerDescriptionChangedEvent): return (False, "Expected ServerDescriptionChangedEvent, got %s" % ( actual.__class__)) if not expected['address'] == "%s:%s" % actual.server_address: return (False, "ServerDescriptionChangedEvent has wrong address" " (expected %s, got %s" % (expected['address'], actual.server_address)) if not compare_server_descriptions( expected['newDescription'], actual.new_description): return (False, "New ServerDescription incorrect in" " ServerDescriptionChangedEvent") if not compare_server_descriptions(expected['previousDescription'], actual.previous_description): return (False, "Previous ServerDescription incorrect in" " ServerDescriptionChangedEvent") elif expected_type == "server_closed_event": if not isinstance(actual, monitoring.ServerClosedEvent): return False, "Expected ServerClosedEvent, got %s" % ( actual.__class__) if not expected['address'] == "%s:%s" % actual.server_address: return (False, "ServerClosedEvent published with wrong address" " (expected %s, got %s" % (expected['address'], actual.server_address)) elif expected_type == "topology_opening_event": if not isinstance(actual, monitoring.TopologyOpenedEvent): return False, "Expected TopologyOpeningEvent, got %s" % ( actual.__class__) elif expected_type == "topology_description_changed_event": if not isinstance(actual, monitoring.TopologyDescriptionChangedEvent): return (False, "Expected TopologyDescriptionChangedEvent," " got %s" % (actual.__class__)) if not compare_topology_descriptions(expected['newDescription'], actual.new_description): return (False, "New TopologyDescription incorrect in " "TopologyDescriptionChangedEvent") if not compare_topology_descriptions( expected['previousDescription'], actual.previous_description): return (False, "Previous TopologyDescription incorrect in" " TopologyDescriptionChangedEvent") elif expected_type == "topology_closed_event": if not isinstance(actual, monitoring.TopologyClosedEvent): return False, "Expected TopologyClosedEvent, got %s" % ( actual.__class__) else: return False, "Incorrect event: expected %s, actual %s" % ( expected_type, actual) return True, "" def compare_multiple_events(i, expected_results, actual_results): events_in_a_row = [] j = i while(j < len(expected_results) and isinstance( actual_results[j], actual_results[i].__class__)): events_in_a_row.append(actual_results[j]) j += 1 message = '' for event in events_in_a_row: for k in range(i, j): passed, message = compare_events(expected_results[k], event) if passed: expected_results[k] = None break else: return i, False, message return j, True, '' class TestAllScenarios(unittest.TestCase): @classmethod @client_context.require_connection def setUp(cls): cls.all_listener = ServerAndTopologyEventListener() cls.saved_listeners = monitoring._LISTENERS monitoring._LISTENERS = monitoring._Listeners([], [], [], []) @classmethod def tearDown(cls): monitoring._LISTENERS = cls.saved_listeners def create_test(scenario_def): def run_scenario(self): responses = (r for r in scenario_def['phases'][0]['responses']) with client_knobs(events_queue_frequency=0.1): class MockMonitor(Monitor): def __init__(self, server_description, topology, pool, topology_settings): """Have to copy entire constructor from Monitor so that we can override _run and change the periodic executor's interval.""" self._server_description = server_description self._pool = pool self._settings = topology_settings self._avg_round_trip_time = MovingAverage() options = self._settings._pool_options self._listeners = options.event_listeners self._publish = self._listeners is not None def target(): monitor = self_ref() if monitor is None: return False MockMonitor._run(monitor) # Change target to subclass return True # Shorten interval executor = periodic_executor.PeriodicExecutor( interval=0.1, min_interval=0.1, target=target, name="pymongo_server_monitor_thread") self._executor = executor self_ref = weakref.ref(self, executor.close) self._topology = weakref.proxy(topology, executor.close) def _run(self): try: if self._server_description.address != ('a', 27017): # Because PyMongo doesn't keep information about # the order of addresses, we might accidentally # start a MockMonitor on the wrong server first, # so we need to only mock responses for the server # the test's response is supposed to come from. return response = next(responses)[1] isMaster = IsMaster(response) self._server_description = ServerDescription( address=self._server_description.address, ismaster=isMaster) self._topology.on_change(self._server_description) except (ReferenceError, StopIteration): # Topology was garbage-collected. self.close() m = single_client(h=scenario_def['uri'], p=27017, event_listeners=(self.all_listener,), _monitor_class=MockMonitor) expected_results = scenario_def['phases'][0]['outcome']['events'] expected_len = len(expected_results) wait_until(lambda: len(self.all_listener.results) >= expected_len, "publish all events", timeout=15) try: i = 0 while i < expected_len: result = self.all_listener.results[i] if len( self.all_listener.results) > i else None # The order of ServerOpening/ClosedEvents doesn't matter if (isinstance(result, monitoring.ServerOpeningEvent) or isinstance(result, monitoring.ServerClosedEvent)): i, passed, message = compare_multiple_events( i, expected_results, self.all_listener.results) self.assertTrue(passed, message) else: self.assertTrue( *compare_events(expected_results[i], result)) i += 1 finally: m.close() return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json.load( scenario_stream, object_hook=object_hook) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s' % (os.path.splitext(filename)[0],) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_replica_set_client.py0000644000076600000240000003234313245621354021665 0ustar shanestaff00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the mongo_replica_set_client module.""" import sys import warnings import time sys.path[0:0] = [""] from bson.codec_options import CodecOptions from bson.son import SON from pymongo.common import MAX_SUPPORTED_WIRE_VERSION, partition_node from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure, NetworkTimeout, NotMasterError, OperationFailure) from pymongo.mongo_client import MongoClient from pymongo.mongo_replica_set_client import MongoReplicaSetClient from pymongo.read_preferences import ReadPreference, Secondary, Nearest from pymongo.write_concern import WriteConcern from test import (client_context, client_knobs, IntegrationTest, unittest, SkipTest, db_pwd, db_user, MockClientTest, HAVE_IPADDRESS) from test.pymongo_mocks import MockClient from test.utils import (connected, delay, ignore_deprecations, one, rs_client, single_client, wait_until) class TestReplicaSetClientBase(IntegrationTest): @classmethod @client_context.require_replica_set def setUpClass(cls): super(TestReplicaSetClientBase, cls).setUpClass() cls.name = client_context.replica_set_name cls.w = client_context.w ismaster = client_context.ismaster cls.hosts = set(partition_node(h.lower()) for h in ismaster['hosts']) cls.arbiters = set(partition_node(h) for h in ismaster.get("arbiters", [])) repl_set_status = client_context.client.admin.command( 'replSetGetStatus') primary_info = [ m for m in repl_set_status['members'] if m['stateStr'] == 'PRIMARY' ][0] cls.primary = partition_node(primary_info['name'].lower()) cls.secondaries = set( partition_node(m['name'].lower()) for m in repl_set_status['members'] if m['stateStr'] == 'SECONDARY') class TestReplicaSetClient(TestReplicaSetClientBase): def test_deprecated(self): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) with self.assertRaises(DeprecationWarning): MongoReplicaSetClient() def test_connect(self): client = MongoClient( client_context.pair, replicaSet='fdlksjfdslkjfd', serverSelectionTimeoutMS=100) with self.assertRaises(ConnectionFailure): client.test.test.find_one() def test_repr(self): with ignore_deprecations(): client = MongoReplicaSetClient( client_context.host, client_context.port, replicaSet=self.name) self.assertIn("MongoReplicaSetClient(host=[", repr(client)) self.assertIn(client_context.pair, repr(client)) def test_properties(self): c = client_context.client c.admin.command('ping') wait_until(lambda: c.primary == self.primary, "discover primary") wait_until(lambda: c.secondaries == self.secondaries, "discover secondaries") # SERVER-32845 if not (client_context.version >= (3, 7, 2) and client_context.auth_enabled and client_context.is_rs): wait_until(lambda: c.arbiters == self.arbiters, "discover arbiters") self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.primary, self.primary) self.assertEqual(c.secondaries, self.secondaries) self.assertEqual(c.max_pool_size, 100) # Make sure MongoClient's properties are copied to Database and # Collection. for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.codec_options, CodecOptions()) self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.write_concern, WriteConcern()) cursor = c.pymongo_test.test.find() self.assertEqual( ReadPreference.PRIMARY, cursor._Cursor__read_preference) tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}] secondary = Secondary(tag_sets=tag_sets) c = rs_client( maxPoolSize=25, document_class=SON, tz_aware=True, read_preference=secondary, localThresholdMS=77, j=True) self.assertEqual(c.max_pool_size, 25) for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.codec_options, CodecOptions(SON, True)) self.assertEqual(obj.read_preference, secondary) self.assertEqual(obj.write_concern, WriteConcern(j=True)) cursor = c.pymongo_test.test.find() self.assertEqual( secondary, cursor._Cursor__read_preference) nearest = Nearest(tag_sets=[{'dc': 'ny'}, {}]) cursor = c.pymongo_test.get_collection( "test", read_preference=nearest).find() self.assertEqual(nearest, cursor._Cursor__read_preference) self.assertEqual(c.max_bson_size, 16777216) c.close() @client_context.require_secondaries_count(1) def test_timeout_does_not_mark_member_down(self): # If a query times out, the client shouldn't mark the member "down". # Disable background refresh. with client_knobs(heartbeat_frequency=999999): c = rs_client(socketTimeoutMS=3000, w=self.w) collection = c.pymongo_test.test collection.insert_one({}) # Query the primary. self.assertRaises( NetworkTimeout, collection.find_one, {'$where': delay(5)}) self.assertTrue(c.primary) collection.find_one() # No error. coll = collection.with_options( read_preference=ReadPreference.SECONDARY) # Query the secondary. self.assertRaises( NetworkTimeout, coll.find_one, {'$where': delay(5)}) self.assertTrue(c.secondaries) # No error. coll.find_one() @client_context.require_ipv6 def test_ipv6(self): if client_context.ssl: # http://bugs.python.org/issue13034 if sys.version_info[:2] == (2, 6): raise SkipTest("Python 2.6 can't parse SANs") if not HAVE_IPADDRESS: raise SkipTest("Need the ipaddress module to test with SSL") port = client_context.port c = rs_client("mongodb://[::1]:%d" % (port,)) # Client switches to IPv4 once it has first ismaster response. msg = 'discovered primary with IPv4 address "%r"' % (self.primary,) wait_until(lambda: c.primary == self.primary, msg) # Same outcome with both IPv4 and IPv6 seeds. c = rs_client("mongodb://[::1]:%d,localhost:%d" % (port, port)) wait_until(lambda: c.primary == self.primary, msg) if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) else: auth_str = "" uri = "mongodb://%slocalhost:%d,[::1]:%d" % (auth_str, port, port) client = rs_client(uri) client.pymongo_test.test.insert_one({"dummy": u"object"}) client.pymongo_test_bernie.test.insert_one({"dummy": u"object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_bernie" in dbs) client.close() def _test_kill_cursor_explicit(self, read_pref): with client_knobs(kill_cursor_frequency=0.01): c = rs_client(read_preference=read_pref, w=self.w) db = c.pymongo_test db.drop_collection("test") test = db.test test.insert_many([{"i": i} for i in range(20)]) # Partially evaluate cursor so it's left alive, then kill it cursor = test.find().batch_size(10) next(cursor) self.assertNotEqual(0, cursor.cursor_id) if read_pref == ReadPreference.PRIMARY: msg = "Expected cursor's address to be %s, got %s" % ( c.primary, cursor.address) self.assertEqual(cursor.address, c.primary, msg) else: self.assertNotEqual( cursor.address, c.primary, "Expected cursor's address not to be primary") cursor_id = cursor.cursor_id # Cursor dead on server - trigger a getMore on the same cursor_id # and check that the server returns an error. cursor2 = cursor.clone() cursor2._Cursor__id = cursor_id if sys.platform.startswith('java') or 'PyPy' in sys.version: # Explicitly kill cursor. cursor.close() else: # Implicitly kill it in CPython. del cursor time.sleep(5) self.assertRaises(OperationFailure, lambda: list(cursor2)) def test_kill_cursor_explicit_primary(self): self._test_kill_cursor_explicit(ReadPreference.PRIMARY) @client_context.require_secondaries_count(1) def test_kill_cursor_explicit_secondary(self): self._test_kill_cursor_explicit(ReadPreference.SECONDARY) @client_context.require_secondaries_count(1) def test_not_master_error(self): secondary_address = one(self.secondaries) direct_client = single_client(*secondary_address) with self.assertRaises(NotMasterError): direct_client.pymongo_test.collection.insert_one({}) db = direct_client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)) with self.assertRaises(NotMasterError): db.collection.insert_one({}) class TestReplicaSetWireVersion(MockClientTest): @client_context.require_connection @client_context.require_no_auth def test_wire_version(self): c = MockClient( standalones=[], members=['a:1', 'b:2', 'c:3'], mongoses=[], host='a:1', replicaSet='rs', connect=False) c.set_wire_version_range('a:1', 3, 7) c.set_wire_version_range('b:2', 2, 3) c.set_wire_version_range('c:3', 3, 4) c.db.command('ismaster') # Connect. # A secondary doesn't overlap with us. c.set_wire_version_range('b:2', MAX_SUPPORTED_WIRE_VERSION + 1, MAX_SUPPORTED_WIRE_VERSION + 2) def raises_configuration_error(): try: c.db.collection.find_one() return False except ConfigurationError: return True wait_until(raises_configuration_error, 'notice we are incompatible with server') self.assertRaises(ConfigurationError, c.db.collection.insert_one, {}) class TestReplicaSetClientInternalIPs(MockClientTest): @client_context.require_connection def test_connect_with_internal_ips(self): # Client is passed an IP it can reach, 'a:1', but the RS config # only contains unreachable IPs like 'internal-ip'. PYTHON-608. with self.assertRaises(AutoReconnect) as context: connected(MockClient( standalones=[], members=['a:1'], mongoses=[], ismaster_hosts=['internal-ip:27017'], host='a:1', replicaSet='rs', serverSelectionTimeoutMS=100)) self.assertEqual( "Could not reach any servers in [('internal-ip', 27017)]." " Replica set is configured with internal hostnames or IPs?", str(context.exception)) class TestReplicaSetClientMaxWriteBatchSize(MockClientTest): @client_context.require_connection def test_max_write_batch_size(self): c = MockClient( standalones=[], members=['a:1', 'b:2'], mongoses=[], host='a:1', replicaSet='rs', connect=False) c.set_max_write_batch_size('a:1', 1) c.set_max_write_batch_size('b:2', 2) # Uses primary's max batch size. self.assertEqual(c.max_write_batch_size, 1) # b becomes primary. c.mock_primary = 'b:2' wait_until(lambda: c.max_write_batch_size == 2, 'update max_write_batch_size') if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_client.py0000644000076600000240000016545613245621354017327 0ustar shanestaff00000000000000# Copyright 2013-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the mongo_client module.""" import contextlib import datetime import gc import os import signal import socket import struct import sys import time import warnings sys.path[0:0] = [""] from bson import BSON from bson.codec_options import CodecOptions from bson.py3compat import thread from bson.son import SON from bson.tz_util import utc from pymongo import auth, message from pymongo.common import _UUID_REPRESENTATIONS from pymongo.command_cursor import CommandCursor from pymongo.cursor import CursorType from pymongo.database import Database from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure, InvalidName, InvalidURI, NetworkTimeout, OperationFailure, WriteConcernError) from pymongo.monitoring import (ServerHeartbeatListener, ServerHeartbeatStartedEvent) from pymongo.mongo_client import MongoClient from pymongo.pool import SocketInfo, _METADATA from pymongo.read_preferences import ReadPreference from pymongo.server_selectors import (any_server_selector, writable_server_selector) from pymongo.server_type import SERVER_TYPE from pymongo.write_concern import WriteConcern from test import (client_context, client_knobs, SkipTest, unittest, IntegrationTest, db_pwd, db_user, MockClientTest, HAVE_IPADDRESS) from test.pymongo_mocks import MockClient from test.utils import (assertRaisesExactly, connected, delay, get_pool, gevent_monkey_patched, ignore_deprecations, is_greenthread_patched, lazy_client_trial, NTHREADS, one, remove_all_users, rs_client, rs_or_single_client, rs_or_single_client_noauth, server_is_master_with_slave, single_client, wait_until) class ClientUnitTest(unittest.TestCase): """MongoClient tests that don't require a server.""" @classmethod @client_context.require_connection def setUpClass(cls): cls.client = rs_or_single_client(connect=False, serverSelectionTimeoutMS=100) def test_keyword_arg_defaults(self): client = MongoClient(socketTimeoutMS=None, connectTimeoutMS=20000, waitQueueTimeoutMS=None, waitQueueMultiple=None, replicaSet=None, read_preference=ReadPreference.PRIMARY, ssl=False, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=0, # ssl.CERT_NONE ssl_ca_certs=None, connect=False, serverSelectionTimeoutMS=12000) options = client._MongoClient__options pool_opts = options.pool_options self.assertEqual(None, pool_opts.socket_timeout) # socket.Socket.settimeout takes a float in seconds self.assertEqual(20.0, pool_opts.connect_timeout) self.assertEqual(None, pool_opts.wait_queue_timeout) self.assertEqual(None, pool_opts.wait_queue_multiple) self.assertTrue(pool_opts.socket_keepalive) self.assertEqual(None, pool_opts.ssl_context) self.assertEqual(None, options.replica_set_name) self.assertEqual(ReadPreference.PRIMARY, client.read_preference) self.assertAlmostEqual(12, client.server_selection_timeout) def test_types(self): self.assertRaises(TypeError, MongoClient, 1) self.assertRaises(TypeError, MongoClient, 1.14) self.assertRaises(TypeError, MongoClient, "localhost", "27017") self.assertRaises(TypeError, MongoClient, "localhost", 1.14) self.assertRaises(TypeError, MongoClient, "localhost", []) self.assertRaises(ConfigurationError, MongoClient, []) def test_max_pool_size_zero(self): with self.assertRaises(ValueError): MongoClient(maxPoolSize=0) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.client, "") self.assertRaises(InvalidName, make_db, self.client, "te$t") self.assertRaises(InvalidName, make_db, self.client, "te.t") self.assertRaises(InvalidName, make_db, self.client, "te\\t") self.assertRaises(InvalidName, make_db, self.client, "te/t") self.assertRaises(InvalidName, make_db, self.client, "te st") self.assertTrue(isinstance(self.client.test, Database)) self.assertEqual(self.client.test, self.client["test"]) self.assertEqual(self.client.test, Database(self.client, "test")) def test_get_database(self): codec_options = CodecOptions(tz_aware=True) write_concern = WriteConcern(w=2, j=True) db = self.client.get_database( 'foo', codec_options, ReadPreference.SECONDARY, write_concern) self.assertEqual('foo', db.name) self.assertEqual(codec_options, db.codec_options) self.assertEqual(ReadPreference.SECONDARY, db.read_preference) self.assertEqual(write_concern, db.write_concern) def test_getattr(self): self.assertTrue(isinstance(self.client['_does_not_exist'], Database)) with self.assertRaises(AttributeError) as context: self.client._does_not_exist # Message should be: # "AttributeError: MongoClient has no attribute '_does_not_exist'. To # access the _does_not_exist database, use client['_does_not_exist']". self.assertIn("has no attribute '_does_not_exist'", str(context.exception)) def test_iteration(self): def iterate(): [a for a in self.client] self.assertRaises(TypeError, iterate) def test_get_database_default(self): c = rs_or_single_client("mongodb://%s:%d/foo" % (client_context.host, client_context.port), connect=False) self.assertEqual(Database(c, 'foo'), c.get_database()) def test_get_database_default_error(self): # URI with no database. c = rs_or_single_client("mongodb://%s:%d/" % (client_context.host, client_context.port), connect=False) self.assertRaises(ConfigurationError, c.get_database) def test_get_database_default_with_authsource(self): # Ensure we distinguish database name from authSource. uri = "mongodb://%s:%d/foo?authSource=src" % ( client_context.host, client_context.port) c = rs_or_single_client(uri, connect=False) self.assertEqual(Database(c, 'foo'), c.get_database()) def test_primary_read_pref_with_tags(self): # No tags allowed with "primary". with self.assertRaises(ConfigurationError): MongoClient('mongodb://host/?readpreferencetags=dc:east') with self.assertRaises(ConfigurationError): MongoClient('mongodb://host/?' 'readpreference=primary&readpreferencetags=dc:east') def test_read_preference(self): c = rs_or_single_client( "mongodb://host", connect=False, readpreference=ReadPreference.NEAREST.mongos_mode) self.assertEqual(c.read_preference, ReadPreference.NEAREST) def test_metadata(self): metadata = _METADATA.copy() metadata['application'] = {'name': 'foobar'} client = MongoClient( "mongodb://foo:27017/?appname=foobar&connect=false") options = client._MongoClient__options self.assertEqual(options.pool_options.metadata, metadata) client = MongoClient('foo', 27017, appname='foobar', connect=False) options = client._MongoClient__options self.assertEqual(options.pool_options.metadata, metadata) # No error MongoClient(appname='x' * 128) self.assertRaises(ValueError, MongoClient, appname='x' * 129) def test_kwargs_codec_options(self): # Ensure codec options are passed in correctly document_class = SON tz_aware = True uuid_representation_label = 'javaLegacy' unicode_decode_error_handler = 'ignore' tzinfo = utc c = MongoClient( document_class=document_class, tz_aware=tz_aware, uuidrepresentation=uuid_representation_label, unicode_decode_error_handler=unicode_decode_error_handler, tzinfo=tzinfo, connect=False ) self.assertEqual(c.codec_options.document_class, document_class) self.assertEqual(c.codec_options.tz_aware, tz_aware) self.assertEqual( c.codec_options.uuid_representation, _UUID_REPRESENTATIONS[uuid_representation_label]) self.assertEqual( c.codec_options.unicode_decode_error_handler, unicode_decode_error_handler) self.assertEqual(c.codec_options.tzinfo, tzinfo) def test_uri_codec_options(self): # Ensure codec options are passed in correctly uuid_representation_label = 'javaLegacy' unicode_decode_error_handler = 'ignore' uri = ("mongodb://%s:%d/foo?tz_aware=true&uuidrepresentation=" "%s&unicode_decode_error_handler=%s" % ( client_context.host, client_context.port, uuid_representation_label, unicode_decode_error_handler)) c = MongoClient(uri, connect=False) self.assertEqual(c.codec_options.tz_aware, True) self.assertEqual( c.codec_options.uuid_representation, _UUID_REPRESENTATIONS[uuid_representation_label]) self.assertEqual( c.codec_options.unicode_decode_error_handler, unicode_decode_error_handler) class TestClient(IntegrationTest): def test_max_idle_time_reaper(self): with client_knobs(kill_cursor_frequency=0.1): # Assert reaper doesn't remove sockets when maxIdleTimeMS not set client = rs_or_single_client() server = client._get_topology().select_server(any_server_selector) with server._pool.get_socket({}) as sock_info: pass self.assertEqual(1, len(server._pool.sockets)) self.assertTrue(sock_info in server._pool.sockets) client.close() # Assert reaper removes idle socket and replaces it with a new one client = rs_or_single_client(maxIdleTimeMS=500, minPoolSize=1) server = client._get_topology().select_server(any_server_selector) with server._pool.get_socket({}) as sock_info: pass # When the reaper runs at the same time as the get_socket, two # sockets could be created and checked into the pool. self.assertGreaterEqual(len(server._pool.sockets), 1) wait_until(lambda: sock_info not in server._pool.sockets, "remove stale socket") wait_until(lambda: 1 <= len(server._pool.sockets), "replace stale socket") client.close() # Assert reaper respects maxPoolSize when adding new sockets. client = rs_or_single_client(maxIdleTimeMS=500, minPoolSize=1, maxPoolSize=1) server = client._get_topology().select_server(any_server_selector) with server._pool.get_socket({}) as sock_info: pass # When the reaper runs at the same time as the get_socket, # maxPoolSize=1 should prevent two sockets from being created. self.assertEqual(1, len(server._pool.sockets)) wait_until(lambda: sock_info not in server._pool.sockets, "remove stale socket") wait_until(lambda: 1 == len(server._pool.sockets), "replace stale socket") client.close() # Assert reaper has removed idle socket and NOT replaced it client = rs_or_single_client(maxIdleTimeMS=500) server = client._get_topology().select_server(any_server_selector) with server._pool.get_socket({}) as sock_info_one: pass # Assert that the pool does not close sockets prematurely. time.sleep(.300) with server._pool.get_socket({}) as sock_info_two: pass self.assertIs(sock_info_one, sock_info_two) wait_until( lambda: 0 == len(server._pool.sockets), "stale socket reaped and new one NOT added to the pool") client.close() def test_min_pool_size(self): with client_knobs(kill_cursor_frequency=.1): client = rs_or_single_client() server = client._get_topology().select_server(any_server_selector) self.assertEqual(0, len(server._pool.sockets)) # Assert that pool started up at minPoolSize client = rs_or_single_client(minPoolSize=10) server = client._get_topology().select_server(any_server_selector) wait_until(lambda: 10 == len(server._pool.sockets), "pool initialized with 10 sockets") # Assert that if a socket is closed, a new one takes its place with server._pool.get_socket({}) as sock_info: sock_info.close() wait_until(lambda: 10 == len(server._pool.sockets), "a closed socket gets replaced from the pool") self.assertFalse(sock_info in server._pool.sockets) def test_max_idle_time_checkout(self): # Use high frequency to test _get_socket_no_auth. with client_knobs(kill_cursor_frequency=99999999): client = rs_or_single_client(maxIdleTimeMS=500) server = client._get_topology().select_server(any_server_selector) with server._pool.get_socket({}) as sock_info: pass self.assertEqual(1, len(server._pool.sockets)) time.sleep(1) # Sleep so that the socket becomes stale. with server._pool.get_socket({}) as new_sock_info: self.assertNotEqual(sock_info, new_sock_info) self.assertEqual(1, len(server._pool.sockets)) self.assertFalse(sock_info in server._pool.sockets) self.assertTrue(new_sock_info in server._pool.sockets) # Test that sockets are reused if maxIdleTimeMS is not set. client = rs_or_single_client() server = client._get_topology().select_server(any_server_selector) with server._pool.get_socket({}) as sock_info: pass self.assertEqual(1, len(server._pool.sockets)) time.sleep(1) with server._pool.get_socket({}) as new_sock_info: self.assertEqual(sock_info, new_sock_info) self.assertEqual(1, len(server._pool.sockets)) def test_constants(self): """This test uses MongoClient explicitly to make sure that host and port are not overloaded. """ host, port = client_context.host, client_context.port kwargs = client_context.ssl_client_options.copy() if client_context.auth_enabled: kwargs['username'] = db_user kwargs['password'] = db_pwd # Set bad defaults. MongoClient.HOST = "somedomainthatdoesntexist.org" MongoClient.PORT = 123456789 with self.assertRaises(AutoReconnect): connected(MongoClient(serverSelectionTimeoutMS=10, **kwargs)) # Override the defaults. No error. connected(MongoClient(host, port, **kwargs)) # Set good defaults. MongoClient.HOST = host MongoClient.PORT = port # No error. connected(MongoClient(**kwargs)) def test_init_disconnected(self): host, port = client_context.host, client_context.port c = rs_or_single_client(connect=False) # is_primary causes client to block until connected self.assertIsInstance(c.is_primary, bool) c = rs_or_single_client(connect=False) self.assertIsInstance(c.is_mongos, bool) c = rs_or_single_client(connect=False) self.assertIsInstance(c.max_pool_size, int) self.assertIsInstance(c.nodes, frozenset) c = rs_or_single_client(connect=False) self.assertEqual(c.codec_options, CodecOptions()) self.assertIsInstance(c.max_bson_size, int) c = rs_or_single_client(connect=False) self.assertFalse(c.primary) self.assertFalse(c.secondaries) c = rs_or_single_client(connect=False) self.assertIsInstance(c.max_write_batch_size, int) if client_context.is_rs: # The primary's host and port are from the replica set config. self.assertIsNotNone(c.address) else: self.assertEqual(c.address, (host, port)) bad_host = "somedomainthatdoesntexist.org" c = MongoClient(bad_host, port, connectTimeoutMS=1, serverSelectionTimeoutMS=10) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) def test_init_disconnected_with_auth(self): uri = "mongodb://user:pass@somedomainthatdoesntexist" c = MongoClient(uri, connectTimeoutMS=1, serverSelectionTimeoutMS=10) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) def test_equality(self): c = connected(rs_or_single_client()) self.assertEqual(client_context.client, c) # Explicitly test inequality self.assertFalse(client_context.client != c) def test_host_w_port(self): with self.assertRaises(ValueError): connected(MongoClient("%s:1234567" % (client_context.host,), connectTimeoutMS=1, serverSelectionTimeoutMS=10)) def test_repr(self): # Used to test 'eval' below. import bson client = MongoClient( 'mongodb://localhost:27017,localhost:27018/?replicaSet=replset' '&connectTimeoutMS=12345&w=1&wtimeoutms=100', connect=False, document_class=SON) the_repr = repr(client) self.assertIn('MongoClient(host=', the_repr) self.assertIn( "document_class=bson.son.SON, " "tz_aware=False, " "connect=False, ", the_repr) self.assertIn("connecttimeoutms=12345", the_repr) self.assertIn("replicaset='replset'", the_repr) self.assertIn("w=1", the_repr) self.assertIn("wtimeoutms=100", the_repr) self.assertEqual(eval(the_repr), client) client = MongoClient("localhost:27017,localhost:27018", replicaSet='replset', connectTimeoutMS=12345, socketTimeoutMS=None, w=1, wtimeoutms=100, connect=False) the_repr = repr(client) self.assertIn('MongoClient(host=', the_repr) self.assertIn( "document_class=dict, " "tz_aware=False, " "connect=False, ", the_repr) self.assertIn("connecttimeoutms=12345", the_repr) self.assertIn("replicaset='replset'", the_repr) self.assertIn("sockettimeoutms=None", the_repr) self.assertIn("w=1", the_repr) self.assertIn("wtimeoutms=100", the_repr) self.assertEqual(eval(the_repr), client) def test_getters(self): if (client_context.version >= (3, 7, 2) and client_context.auth_enabled and client_context.is_rs): raise SkipTest("Disabled due to SERVER-32845") wait_until(lambda: client_context.nodes == self.client.nodes, "find all nodes") def test_list_databases(self): cmd_docs = self.client.admin.command('listDatabases')['databases'] cursor = self.client.list_databases() self.assertIsInstance(cursor, CommandCursor) helper_docs = list(cursor) self.assertTrue(len(helper_docs) > 0) self.assertEqual(helper_docs, cmd_docs) for doc in helper_docs: self.assertIs(type(doc), dict) client = rs_or_single_client(document_class=SON) for doc in client.list_databases(): self.assertIs(type(doc), dict) if client_context.version.at_least(3, 4, 2): self.client.pymongo_test.test.insert_one({}) cursor = self.client.list_databases(filter={"name": "admin"}) docs = list(cursor) self.assertEqual(1, len(docs)) self.assertEqual(docs[0]["name"], "admin") if client_context.version.at_least(3, 4, 3): cursor = self.client.list_databases(nameOnly=True) for doc in cursor: self.assertEqual(["name"], list(doc)) def _test_list_names(self, meth): self.client.pymongo_test.test.insert_one({"dummy": u"object"}) self.client.pymongo_test_mike.test.insert_one({"dummy": u"object"}) cmd_docs = self.client.admin.command("listDatabases")["databases"] cmd_names = [doc["name"] for doc in cmd_docs] db_names = meth() self.assertTrue("pymongo_test" in db_names) self.assertTrue("pymongo_test_mike" in db_names) self.assertEqual(db_names, cmd_names) def test_list_database_names(self): self._test_list_names(self.client.list_database_names) def test_database_names(self): self._test_list_names(self.client.database_names) def test_drop_database(self): self.assertRaises(TypeError, self.client.drop_database, 5) self.assertRaises(TypeError, self.client.drop_database, None) self.client.pymongo_test.test.insert_one({"dummy": u"object"}) self.client.pymongo_test2.test.insert_one({"dummy": u"object"}) dbs = self.client.database_names() self.assertIn("pymongo_test", dbs) self.assertIn("pymongo_test2", dbs) self.client.drop_database("pymongo_test") if client_context.version.at_least(3, 3, 9) and client_context.is_rs: wc_client = rs_or_single_client(w=len(client_context.nodes) + 1) with self.assertRaises(WriteConcernError): wc_client.drop_database('pymongo_test2') self.client.drop_database(self.client.pymongo_test2) raise SkipTest("This test often fails due to SERVER-2329") dbs = self.client.database_names() self.assertNotIn("pymongo_test", dbs) self.assertNotIn("pymongo_test2", dbs) def test_close(self): coll = self.client.pymongo_test.bar self.client.close() self.client.close() coll.count() self.client.close() self.client.close() coll.count() def test_close_kills_cursors(self): if sys.platform.startswith('java'): # We can't figure out how to make this test reliable with Jython. raise SkipTest("Can't test with Jython") # Kill any cursors possibly queued up by previous tests. gc.collect() self.client._process_periodic_tasks() # Add some test data. coll = self.client.pymongo_test.test_close_kills_cursors docs_inserted = 1000 coll.insert_many([{"i": i} for i in range(docs_inserted)]) # Open a cursor and leave it open on the server. cursor = coll.find().batch_size(10) self.assertTrue(bool(next(cursor))) self.assertLess(cursor.retrieved, docs_inserted) # Open a command cursor and leave it open on the server. cursor = coll.aggregate([], batchSize=10) self.assertTrue(bool(next(cursor))) del cursor # Required for PyPy, Jython and other Python implementations that # don't use reference counting garbage collection. gc.collect() # Close the client and ensure the topology is closed. self.assertTrue(self.client._topology._opened) self.client.close() self.assertFalse(self.client._topology._opened) # The killCursors task should not need to re-open the topology. self.client._process_periodic_tasks() self.assertFalse(self.client._topology._opened) def test_bad_uri(self): with self.assertRaises(InvalidURI): MongoClient("http://localhost") @client_context.require_auth def test_auth_from_uri(self): host, port = client_context.host, client_context.port client_context.create_user("admin", "admin", "pass") self.addCleanup(client_context.drop_user, "admin", "admin") self.addCleanup(remove_all_users, self.client.pymongo_test) client_context.create_user( "pymongo_test", "user", "pass", roles=['userAdmin', 'readWrite']) with self.assertRaises(OperationFailure): connected(rs_or_single_client( "mongodb://a:b@%s:%d" % (host, port))) # No error. connected(rs_or_single_client_noauth( "mongodb://admin:pass@%s:%d" % (host, port))) # Wrong database. uri = "mongodb://admin:pass@%s:%d/pymongo_test" % (host, port) with self.assertRaises(OperationFailure): connected(rs_or_single_client(uri)) # No error. connected(rs_or_single_client_noauth( "mongodb://user:pass@%s:%d/pymongo_test" % (host, port))) # Auth with lazy connection. rs_or_single_client_noauth( "mongodb://user:pass@%s:%d/pymongo_test" % (host, port), connect=False).pymongo_test.test.find_one() # Wrong password. bad_client = rs_or_single_client_noauth( "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port), connect=False) self.assertRaises(OperationFailure, bad_client.pymongo_test.test.find_one) @client_context.require_auth def test_username_and_password(self): client_context.create_user("admin", "ad min", "pa/ss") self.addCleanup(client_context.drop_user, "admin", "ad min") c = rs_or_single_client(username="ad min", password="pa/ss") # Username and password aren't in strings that will likely be logged. self.assertNotIn("ad min", repr(c)) self.assertNotIn("ad min", str(c)) self.assertNotIn("pa/ss", repr(c)) self.assertNotIn("pa/ss", str(c)) # Auth succeeds. c.server_info() with self.assertRaises(OperationFailure): rs_or_single_client(username="ad min", password="foo").server_info() @client_context.require_auth @ignore_deprecations def test_multiple_logins(self): client_context.create_user( 'pymongo_test', 'user1', 'pass', roles=['readWrite']) client_context.create_user( 'pymongo_test', 'user2', 'pass', roles=['readWrite']) self.addCleanup(remove_all_users, self.client.pymongo_test) client = rs_or_single_client_noauth( "mongodb://user1:pass@%s:%d/pymongo_test" % ( client_context.host, client_context.port)) client.pymongo_test.test.find_one() with self.assertRaises(OperationFailure): # Can't log in to the same database with multiple users. client.pymongo_test.authenticate('user2', 'pass') client.pymongo_test.test.find_one() client.pymongo_test.logout() with self.assertRaises(OperationFailure): client.pymongo_test.test.find_one() client.pymongo_test.authenticate('user2', 'pass') client.pymongo_test.test.find_one() with self.assertRaises(OperationFailure): client.pymongo_test.authenticate('user1', 'pass') client.pymongo_test.test.find_one() @client_context.require_auth def test_lazy_auth_raises_operation_failure(self): lazy_client = rs_or_single_client_noauth( "mongodb://user:wrong@%s/pymongo_test" % (client_context.host,), connect=False) assertRaisesExactly( OperationFailure, lazy_client.test.collection.find_one) @client_context.require_no_ssl def test_unix_socket(self): if not hasattr(socket, "AF_UNIX"): raise SkipTest("UNIX-sockets are not supported on this system") mongodb_socket = '/tmp/mongodb-%d.sock' % (client_context.port,) encoded_socket = ( '%2Ftmp%2F' + 'mongodb-%d.sock' % (client_context.port,)) if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessible") if client_context.auth_enabled: uri = "mongodb://%s:%s@%s" % (db_user, db_pwd, encoded_socket) else: uri = "mongodb://%s" % encoded_socket # Confirm we can do operations via the socket. client = rs_or_single_client(uri) client.pymongo_test.test.insert_one({"dummy": "object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue(mongodb_socket in repr(client)) # Confirm it fails with a missing socket. self.assertRaises( ConnectionFailure, connected, MongoClient("mongodb://%2Ftmp%2Fnon-existent.sock", serverSelectionTimeoutMS=100)) def test_document_class(self): c = self.client db = c.pymongo_test db.test.insert_one({"x": 1}) self.assertEqual(dict, c.codec_options.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c = rs_or_single_client(document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.codec_options.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) def test_timeouts(self): client = rs_or_single_client( connectTimeoutMS=10500, socketTimeoutMS=10500, maxIdleTimeMS=10500, serverSelectionTimeoutMS=10500) self.assertEqual(10.5, get_pool(client).opts.connect_timeout) self.assertEqual(10.5, get_pool(client).opts.socket_timeout) self.assertEqual(10.5, get_pool(client).opts.max_idle_time_seconds) self.assertEqual(10500, client.max_idle_time_ms) self.assertEqual(10.5, client.server_selection_timeout) def test_socket_timeout_ms_validation(self): c = rs_or_single_client(socketTimeoutMS=10 * 1000) self.assertEqual(10, get_pool(c).opts.socket_timeout) c = connected(rs_or_single_client(socketTimeoutMS=None)) self.assertEqual(None, get_pool(c).opts.socket_timeout) self.assertRaises(ValueError, rs_or_single_client, socketTimeoutMS=0) self.assertRaises(ValueError, rs_or_single_client, socketTimeoutMS=-1) self.assertRaises(ValueError, rs_or_single_client, socketTimeoutMS=1e10) self.assertRaises(ValueError, rs_or_single_client, socketTimeoutMS='foo') def test_socket_timeout(self): no_timeout = self.client timeout_sec = 1 timeout = rs_or_single_client(socketTimeoutMS=1000 * timeout_sec) no_timeout.pymongo_test.drop_collection("test") no_timeout.pymongo_test.test.insert_one({"x": 1}) # A $where clause that takes a second longer than the timeout where_func = delay(timeout_sec + 1) def get_x(db): doc = next(db.test.find().where(where_func)) return doc["x"] self.assertEqual(1, get_x(no_timeout.pymongo_test)) self.assertRaises(NetworkTimeout, get_x, timeout.pymongo_test) def test_server_selection_timeout(self): client = MongoClient(serverSelectionTimeoutMS=100, connect=False) self.assertAlmostEqual(0.1, client.server_selection_timeout) client = MongoClient(serverSelectionTimeoutMS=0, connect=False) self.assertAlmostEqual(0, client.server_selection_timeout) self.assertRaises(ValueError, MongoClient, serverSelectionTimeoutMS="foo", connect=False) self.assertRaises(ValueError, MongoClient, serverSelectionTimeoutMS=-1, connect=False) self.assertRaises(ConfigurationError, MongoClient, serverSelectionTimeoutMS=None, connect=False) client = MongoClient( 'mongodb://localhost/?serverSelectionTimeoutMS=100', connect=False) self.assertAlmostEqual(0.1, client.server_selection_timeout) client = MongoClient( 'mongodb://localhost/?serverSelectionTimeoutMS=0', connect=False) self.assertAlmostEqual(0, client.server_selection_timeout) # Test invalid timeout in URI ignored and set to default. client = MongoClient( 'mongodb://localhost/?serverSelectionTimeoutMS=-1', connect=False) self.assertAlmostEqual(30, client.server_selection_timeout) client = MongoClient( 'mongodb://localhost/?serverSelectionTimeoutMS=', connect=False) self.assertAlmostEqual(30, client.server_selection_timeout) def test_waitQueueTimeoutMS(self): client = rs_or_single_client(waitQueueTimeoutMS=2000) self.assertEqual(get_pool(client).opts.wait_queue_timeout, 2) def test_waitQueueMultiple(self): client = rs_or_single_client(maxPoolSize=3, waitQueueMultiple=2) pool = get_pool(client) self.assertEqual(pool.opts.wait_queue_multiple, 2) self.assertEqual(pool._socket_semaphore.waiter_semaphore.counter, 6) def test_socketKeepAlive(self): for socketKeepAlive in [True, False]: with warnings.catch_warnings(record=True) as ctx: warnings.simplefilter("always") client = rs_or_single_client(socketKeepAlive=socketKeepAlive) self.assertIn("The socketKeepAlive option is deprecated", str(ctx[0])) pool = get_pool(client) self.assertEqual(socketKeepAlive, pool.opts.socket_keepalive) with pool.get_socket({}) as sock_info: keepalive = sock_info.sock.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) self.assertEqual(socketKeepAlive, bool(keepalive)) def test_tz_aware(self): self.assertRaises(ValueError, MongoClient, tz_aware='foo') aware = rs_or_single_client(tz_aware=True) naive = self.client aware.pymongo_test.drop_collection("test") now = datetime.datetime.utcnow() aware.pymongo_test.test.insert_one({"x": now}) self.assertEqual(None, naive.pymongo_test.test.find_one()["x"].tzinfo) self.assertEqual(utc, aware.pymongo_test.test.find_one()["x"].tzinfo) self.assertEqual( aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None), naive.pymongo_test.test.find_one()["x"]) @client_context.require_ipv6 def test_ipv6(self): if client_context.ssl: # http://bugs.python.org/issue13034 if sys.version_info[:2] == (2, 6): raise SkipTest("Python 2.6 can't parse SANs") if not HAVE_IPADDRESS: raise SkipTest("Need the ipaddress module to test with SSL") if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) else: auth_str = "" uri = "mongodb://%s[::1]:%d" % (auth_str, client_context.port) if client_context.is_rs: uri += '/?replicaSet=' + client_context.replica_set_name client = rs_or_single_client_noauth(uri) client.pymongo_test.test.insert_one({"dummy": u"object"}) client.pymongo_test_bernie.test.insert_one({"dummy": u"object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_bernie" in dbs) @client_context.require_no_mongos def test_fsync_lock_unlock(self): if server_is_master_with_slave(client_context.client): raise SkipTest('SERVER-7714') self.assertFalse(self.client.is_locked) # async flushing not supported on windows... if sys.platform not in ('cygwin', 'win32'): # Work around async becoming a reserved keyword in Python 3.7 opts = {'async': True} self.client.fsync(**opts) self.assertFalse(self.client.is_locked) self.client.fsync(lock=True) self.assertTrue(self.client.is_locked) locked = True self.client.unlock() for _ in range(5): locked = self.client.is_locked if not locked: break time.sleep(1) self.assertFalse(locked) def test_contextlib(self): client = rs_or_single_client() client.pymongo_test.drop_collection("test") client.pymongo_test.test.insert_one({"foo": "bar"}) # The socket used for the previous commands has been returned to the # pool self.assertEqual(1, len(get_pool(client).sockets)) with contextlib.closing(client): self.assertEqual("bar", client.pymongo_test.test.find_one()["foo"]) self.assertEqual(1, len(get_pool(client).sockets)) self.assertEqual(0, len(get_pool(client).sockets)) with client as client: self.assertEqual("bar", client.pymongo_test.test.find_one()["foo"]) self.assertEqual(0, len(get_pool(client).sockets)) def test_interrupt_signal(self): if sys.platform.startswith('java'): # We can't figure out how to raise an exception on a thread that's # blocked on a socket, whether that's the main thread or a worker, # without simply killing the whole thread in Jython. This suggests # PYTHON-294 can't actually occur in Jython. raise SkipTest("Can't test interrupts in Jython") if is_greenthread_patched(): raise SkipTest("Can't reliably test interrupts with green threads") # Test fix for PYTHON-294 -- make sure MongoClient closes its # socket if it gets an interrupt while waiting to recv() from it. db = self.client.pymongo_test # A $where clause which takes 1.5 sec to execute where = delay(1.5) # Need exactly 1 document so find() will execute its $where clause once db.drop_collection('foo') db.foo.insert_one({'_id': 1}) old_signal_handler = None try: # Platform-specific hacks for raising a KeyboardInterrupt on the # main thread while find() is in-progress: On Windows, SIGALRM is # unavailable so we use a second thread. In our Evergreen setup on # Linux, the thread technique causes an error in the test at # sock.recv(): TypeError: 'int' object is not callable # We don't know what causes this, so we hack around it. if sys.platform == 'win32': def interrupter(): # Raises KeyboardInterrupt in the main thread time.sleep(0.25) thread.interrupt_main() thread.start_new_thread(interrupter, ()) else: # Convert SIGALRM to SIGINT -- it's hard to schedule a SIGINT # for one second in the future, but easy to schedule SIGALRM. def sigalarm(num, frame): raise KeyboardInterrupt old_signal_handler = signal.signal(signal.SIGALRM, sigalarm) signal.alarm(1) raised = False try: # Will be interrupted by a KeyboardInterrupt. next(db.foo.find({'$where': where})) except KeyboardInterrupt: raised = True # Can't use self.assertRaises() because it doesn't catch system # exceptions self.assertTrue(raised, "Didn't raise expected KeyboardInterrupt") # Raises AssertionError due to PYTHON-294 -- Mongo's response to # the previous find() is still waiting to be read on the socket, # so the request id's don't match. self.assertEqual( {'_id': 1}, next(db.foo.find()) ) finally: if old_signal_handler: signal.signal(signal.SIGALRM, old_signal_handler) def test_operation_failure(self): # Ensure MongoClient doesn't close socket after it gets an error # response to getLastError. PYTHON-395. We need a new client here # to avoid race conditions caused by replica set failover or idle # socket reaping. client = single_client() client.pymongo_test.test.find_one() pool = get_pool(client) socket_count = len(pool.sockets) self.assertGreaterEqual(socket_count, 1) old_sock_info = next(iter(pool.sockets)) client.pymongo_test.test.drop() client.pymongo_test.test.insert_one({'_id': 'foo'}) self.assertRaises( OperationFailure, client.pymongo_test.test.insert_one, {'_id': 'foo'}) self.assertEqual(socket_count, len(pool.sockets)) new_sock_info = next(iter(pool.sockets)) self.assertEqual(old_sock_info, new_sock_info) def test_lazy_connect_w0(self): # Ensure that connect-on-demand works when the first operation is # an unacknowledged write. This exercises _writable_max_wire_version(). # Use a separate collection to avoid races where we're still # completing an operation on a collection while the next test begins. client = rs_or_single_client(connect=False, w=0) client.test_lazy_connect_w0.test.insert_one({}) client = rs_or_single_client(connect=False) client.test_lazy_connect_w0.test.update_one({}, {'$set': {'x': 1}}) client = rs_or_single_client(connect=False) client.test_lazy_connect_w0.test.delete_one({}) @client_context.require_no_mongos def test_exhaust_network_error(self): # When doing an exhaust query, the socket stays checked out on success # but must be checked in on error to avoid semaphore leaks. client = rs_or_single_client(maxPoolSize=1) collection = client.pymongo_test.test pool = get_pool(client) pool._check_interval_seconds = None # Never check. # Ensure a socket. connected(client) # Cause a network error. sock_info = one(pool.sockets) sock_info.sock.close() cursor = collection.find(cursor_type=CursorType.EXHAUST) with self.assertRaises(ConnectionFailure): next(cursor) self.assertTrue(sock_info.closed) # The semaphore was decremented despite the error. self.assertTrue(pool._socket_semaphore.acquire(blocking=False)) @client_context.require_auth def test_auth_network_error(self): # Make sure there's no semaphore leak if we get a network error # when authenticating a new socket with cached credentials. # Get a client with one socket so we detect if it's leaked. c = connected(rs_or_single_client(maxPoolSize=1, waitQueueTimeoutMS=1)) # Simulate an authenticate() call on a different socket. credentials = auth._build_credentials_tuple( 'DEFAULT', 'admin', db_user, db_pwd, {}) c._cache_credentials('test', credentials, connect=False) # Cause a network error on the actual socket. pool = get_pool(c) socket_info = one(pool.sockets) socket_info.sock.close() # SocketInfo.check_auth logs in with the new credential, but gets a # socket.error. Should be reraised as AutoReconnect. self.assertRaises(AutoReconnect, c.test.collection.find_one) # No semaphore leak, the pool is allowed to make a new socket. c.test.collection.find_one() @client_context.require_no_replica_set def test_connect_to_standalone_using_replica_set_name(self): client = single_client(replicaSet='anything', serverSelectionTimeoutMS=100) with self.assertRaises(AutoReconnect): client.test.test.find_one() @client_context.require_replica_set def test_stale_getmore(self): # A cursor is created, but its member goes down and is removed from # the topology before the getMore message is sent. Test that # MongoClient._send_message_with_response handles the error. with self.assertRaises(AutoReconnect): client = rs_client(connect=False, serverSelectionTimeoutMS=100) client._send_message_with_response( operation=message._GetMore('pymongo_test', 'collection', 101, 1234, client.codec_options, None, client), address=('not-a-member', 27017)) def test_heartbeat_frequency_ms(self): class HeartbeatStartedListener(ServerHeartbeatListener): def __init__(self): self.results = [] def started(self, event): self.results.append(event) def succeeded(self, event): pass def failed(self, event): pass old_init = ServerHeartbeatStartedEvent.__init__ heartbeat_times = [] def init(self, *args): old_init(self, *args) heartbeat_times.append(time.time()) try: ServerHeartbeatStartedEvent.__init__ = init listener = HeartbeatStartedListener() uri = "mongodb://%s:%d/?heartbeatFrequencyMS=500" % ( client_context.host, client_context.port) client = single_client(uri, event_listeners=[listener]) wait_until(lambda: len(listener.results) >= 2, "record two ServerHeartbeatStartedEvents") # Default heartbeatFrequencyMS is 10 sec. Check the interval was # closer to 0.5 sec with heartbeatFrequencyMS configured. self.assertAlmostEqual( heartbeat_times[1] - heartbeat_times[0], 0.5, delta=2) client.close() finally: ServerHeartbeatStartedEvent.__init__ = old_init def test_small_heartbeat_frequency_ms(self): uri = "mongodb://example/?heartbeatFrequencyMS=499" with self.assertRaises(ConfigurationError) as context: MongoClient(uri) self.assertIn('heartbeatFrequencyMS', str(context.exception)) class TestExhaustCursor(IntegrationTest): """Test that clients properly handle errors from exhaust cursors.""" def setUp(self): super(TestExhaustCursor, self).setUp() if client_context.is_mongos: raise SkipTest("mongos doesn't support exhaust, SERVER-2627") def test_exhaust_query_server_error(self): # When doing an exhaust query, the socket stays checked out on success # but must be checked in on error to avoid semaphore leaks. client = connected(rs_or_single_client(maxPoolSize=1)) collection = client.pymongo_test.test pool = get_pool(client) sock_info = one(pool.sockets) # This will cause OperationFailure in all mongo versions since # the value for $orderby must be a document. cursor = collection.find( SON([('$query', {}), ('$orderby', True)]), cursor_type=CursorType.EXHAUST) self.assertRaises(OperationFailure, cursor.next) self.assertFalse(sock_info.closed) # The socket was checked in and the semaphore was decremented. self.assertIn(sock_info, pool.sockets) self.assertTrue(pool._socket_semaphore.acquire(blocking=False)) def test_exhaust_getmore_server_error(self): # When doing a getmore on an exhaust cursor, the socket stays checked # out on success but it's checked in on error to avoid semaphore leaks. client = rs_or_single_client(maxPoolSize=1) collection = client.pymongo_test.test collection.drop() collection.insert_many([{} for _ in range(200)]) self.addCleanup(client_context.client.pymongo_test.test.drop) pool = get_pool(client) pool._check_interval_seconds = None # Never check. sock_info = one(pool.sockets) cursor = collection.find(cursor_type=CursorType.EXHAUST) # Initial query succeeds. cursor.next() # Cause a server error on getmore. def receive_message(request_id): # Discard the actual server response. SocketInfo.receive_message(sock_info, request_id) # responseFlags bit 1 is QueryFailure. msg = struct.pack(' MAX_ITERATION_TIME: warnings.warn('Test timed out, completed %s iterations.' % i) break self.before() with Timer() as timer: self.do_task() self.after() results.append(timer.interval) self.results = results # BSON MICRO-BENCHMARKS class BsonEncodingTest(PerformanceTest): def setUp(self): # Location of test data. with open( os.path.join(TEST_PATH, os.path.join('extended_bson', self.dataset))) as data: self.document = loads(data.read()) def do_task(self): for _ in range(NUM_DOCS): BSON.encode(self.document) class BsonDecodingTest(PerformanceTest): def setUp(self): # Location of test data. with open( os.path.join(TEST_PATH, os.path.join('extended_bson', self.dataset))) as data: self.document = BSON.encode(json.loads(data.read())) def do_task(self): for _ in range(NUM_DOCS): self.document.decode() class TestFlatEncoding(BsonEncodingTest, unittest.TestCase): dataset = 'flat_bson.json' data_size = 75310000 class TestFlatDecoding(BsonDecodingTest, unittest.TestCase): dataset = 'flat_bson.json' data_size = 75310000 class TestDeepEncoding(BsonEncodingTest, unittest.TestCase): dataset = 'deep_bson.json' data_size = 19640000 class TestDeepDecoding(BsonDecodingTest, unittest.TestCase): dataset = 'deep_bson.json' data_size = 19640000 class TestFullEncoding(BsonEncodingTest, unittest.TestCase): dataset = 'full_bson.json' data_size = 57340000 class TestFullDecoding(BsonDecodingTest, unittest.TestCase): dataset = 'full_bson.json' data_size = 57340000 # SINGLE-DOC BENCHMARKS class TestRunCommand(PerformanceTest, unittest.TestCase): data_size = 160000 def setUp(self): self.client = client_context.client self.client.drop_database('perftest') def do_task(self): command = self.client.perftest.command for _ in range(NUM_DOCS): command("ismaster") class TestDocument(PerformanceTest): def setUp(self): # Location of test data. with open( os.path.join( TEST_PATH, os.path.join( 'single_and_multi_document', self.dataset)), 'r') as data: self.document = json.loads(data.read()) self.client = client_context.client self.client.drop_database('perftest') def tearDown(self): super(TestDocument, self).tearDown() self.client.drop_database('perftest') def before(self): self.corpus = self.client.perftest.create_collection('corpus') def after(self): self.client.perftest.drop_collection('corpus') class TestFindOneByID(TestDocument, unittest.TestCase): data_size = 16220000 def setUp(self): self.dataset = 'tweet.json' super(TestFindOneByID, self).setUp() documents = [self.document.copy() for _ in range(NUM_DOCS)] self.corpus = self.client.perftest.corpus result = self.corpus.insert_many(documents) self.inserted_ids = result.inserted_ids def do_task(self): find_one = self.corpus.find_one for _id in self.inserted_ids: find_one({'_id': _id}) def before(self): pass def after(self): pass class TestSmallDocInsertOne(TestDocument, unittest.TestCase): data_size = 2750000 def setUp(self): self.dataset = 'small_doc.json' super(TestSmallDocInsertOne, self).setUp() self.documents = [self.document.copy() for _ in range(NUM_DOCS)] def do_task(self): insert_one = self.corpus.insert_one for doc in self.documents: insert_one(doc) class TestLargeDocInsertOne(TestDocument, unittest.TestCase): data_size = 27310890 def setUp(self): self.dataset = 'large_doc.json' super(TestLargeDocInsertOne, self).setUp() self.documents = [self.document.copy() for _ in range(10)] def do_task(self): insert_one = self.corpus.insert_one for doc in self.documents: insert_one(doc) # MULTI-DOC BENCHMARKS class TestFindManyAndEmptyCursor(TestDocument, unittest.TestCase): data_size = 16220000 def setUp(self): self.dataset = 'tweet.json' super(TestFindManyAndEmptyCursor, self).setUp() for _ in range(10): self.client.perftest.command( 'insert', 'corpus', documents=[self.document] * 1000) self.corpus = self.client.perftest.corpus def do_task(self): list(self.corpus.find()) def before(self): pass def after(self): pass class TestSmallDocBulkInsert(TestDocument, unittest.TestCase): data_size = 2750000 def setUp(self): self.dataset = 'small_doc.json' super(TestSmallDocBulkInsert, self).setUp() self.documents = [self.document.copy() for _ in range(NUM_DOCS)] def before(self): self.corpus = self.client.perftest.create_collection('corpus') def do_task(self): self.corpus.insert_many(self.documents, ordered=True) class TestLargeDocBulkInsert(TestDocument, unittest.TestCase): data_size = 27310890 def setUp(self): self.dataset = 'large_doc.json' super(TestLargeDocBulkInsert, self).setUp() self.documents = [self.document.copy() for _ in range(10)] def before(self): self.corpus = self.client.perftest.create_collection('corpus') def do_task(self): self.corpus.insert_many(self.documents, ordered=True) class TestGridFsUpload(PerformanceTest, unittest.TestCase): data_size = 52428800 def setUp(self): self.client = client_context.client self.client.drop_database('perftest') gridfs_path = os.path.join( TEST_PATH, os.path.join('single_and_multi_document', 'gridfs_large.bin')) with open(gridfs_path, 'rb') as data: self.document = data.read() self.bucket = GridFSBucket(self.client.perftest) def tearDown(self): super(TestGridFsUpload, self).tearDown() self.client.drop_database('perftest') def before(self): self.bucket.upload_from_stream('init', b'x') def do_task(self): self.bucket.upload_from_stream('gridfstest', self.document) class TestGridFsDownload(PerformanceTest, unittest.TestCase): data_size = 52428800 def setUp(self): self.client = client_context.client self.client.drop_database('perftest') gridfs_path = os.path.join( TEST_PATH, os.path.join('single_and_multi_document', 'gridfs_large.bin')) self.bucket = GridFSBucket(self.client.perftest) with open(gridfs_path, 'rb') as gfile: self.uploaded_id = self.bucket.upload_from_stream( 'gridfstest', gfile) def tearDown(self): super(TestGridFsDownload, self).tearDown() self.client.drop_database('perftest') def do_task(self): self.bucket.open_download_stream(self.uploaded_id).read() proc_client = None def proc_init(*dummy): global proc_client proc_client = MongoClient(host, port) # PARALLEL BENCHMARKS def mp_map(map_func, files): pool = mp.Pool(initializer=proc_init) pool.map(map_func, files) pool.close() def insert_json_file(filename): with open(filename, 'r') as data: coll = proc_client.perftest.corpus coll.insert_many([json.loads(line) for line in data]) def insert_json_file_with_file_id(filename): documents = [] with open(filename, 'r') as data: for line in data: doc = json.loads(line) doc['file'] = filename documents.append(doc) coll = proc_client.perftest.corpus coll.insert_many(documents) def read_json_file(filename): coll = proc_client.perftest.corpus temp = tempfile.TemporaryFile() try: temp.writelines( [json.dumps(doc) + '\n' for doc in coll.find({'file': filename}, {'_id': False})]) finally: temp.close() def insert_gridfs_file(filename): bucket = GridFSBucket(proc_client.perftest) with open(filename, 'rb') as gfile: bucket.upload_from_stream(filename, gfile) def read_gridfs_file(filename): bucket = GridFSBucket(proc_client.perftest) temp = tempfile.TemporaryFile() try: bucket.download_to_stream_by_name(filename, temp) finally: temp.close() class TestJsonMultiImport(PerformanceTest, unittest.TestCase): data_size = 565000000 def setUp(self): self.client = client_context.client self.client.drop_database('perftest') def before(self): self.client.perftest.command({'create': 'corpus'}) self.corpus = self.client.perftest.corpus ldjson_path = os.path.join( TEST_PATH, os.path.join('parallel', 'ldjson_multi')) self.files = [os.path.join( ldjson_path, s) for s in os.listdir(ldjson_path)] def do_task(self): mp_map(insert_json_file, self.files) def after(self): self.client.perftest.drop_collection('corpus') def tearDown(self): super(TestJsonMultiImport, self).tearDown() self.client.drop_database('perftest') class TestJsonMultiExport(PerformanceTest, unittest.TestCase): data_size = 565000000 def setUp(self): self.client = client_context.client self.client.drop_database('perftest') self.client.perfest.corpus.create_index('file') ldjson_path = os.path.join( TEST_PATH, os.path.join('parallel', 'ldjson_multi')) self.files = [os.path.join( ldjson_path, s) for s in os.listdir(ldjson_path)] mp_map(insert_json_file_with_file_id, self.files) def do_task(self): mp_map(read_json_file, self.files) def tearDown(self): super(TestJsonMultiExport, self).tearDown() self.client.drop_database('perftest') class TestGridFsMultiFileUpload(PerformanceTest, unittest.TestCase): data_size = 262144000 def setUp(self): self.client = client_context.client self.client.drop_database('perftest') def before(self): self.client.perftest.drop_collection('fs.files') self.client.perftest.drop_collection('fs.chunks') self.bucket = GridFSBucket(self.client.perftest) gridfs_path = os.path.join( TEST_PATH, os.path.join('parallel', 'gridfs_multi')) self.files = [os.path.join( gridfs_path, s) for s in os.listdir(gridfs_path)] def do_task(self): mp_map(insert_gridfs_file, self.files) def tearDown(self): super(TestGridFsMultiFileUpload, self).tearDown() self.client.drop_database('perftest') class TestGridFsMultiFileDownload(PerformanceTest, unittest.TestCase): data_size = 262144000 def setUp(self): self.client = client_context.client self.client.drop_database('perftest') bucket = GridFSBucket(self.client.perftest) gridfs_path = os.path.join( TEST_PATH, os.path.join('parallel', 'gridfs_multi')) self.files = [os.path.join( gridfs_path, s) for s in os.listdir(gridfs_path)] for fname in self.files: with open(fname, 'rb') as gfile: bucket.upload_from_stream(fname, gfile) def do_task(self): mp_map(read_gridfs_file, self.files) def tearDown(self): super(TestGridFsMultiFileDownload, self).tearDown() self.client.drop_database('perftest') if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_collation.py0000644000076600000240000004023613245621354020021 0ustar shanestaff00000000000000# Copyright 2016-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the collation module.""" import functools import warnings from test import unittest, client_context from test.utils import EventListener, rs_or_single_client from pymongo import monitoring from pymongo.collation import ( Collation, CollationCaseFirst, CollationStrength, CollationAlternate, CollationMaxVariable) from pymongo.errors import ConfigurationError from pymongo.write_concern import WriteConcern from pymongo.operations import (DeleteMany, DeleteOne, IndexModel, ReplaceOne, UpdateMany, UpdateOne) class TestCollationObject(unittest.TestCase): def test_constructor(self): self.assertRaises(TypeError, Collation, locale=42) # Fill in a locale to test the other options. _Collation = functools.partial(Collation, 'en_US') # No error. _Collation(caseFirst=CollationCaseFirst.UPPER) self.assertRaises(TypeError, _Collation, caseLevel='true') self.assertRaises(ValueError, _Collation, strength='six') self.assertRaises(TypeError, _Collation, numericOrdering='true') self.assertRaises(TypeError, _Collation, alternate=5) self.assertRaises(TypeError, _Collation, maxVariable=2) self.assertRaises(TypeError, _Collation, normalization='false') self.assertRaises(TypeError, _Collation, backwards='true') # No errors. Collation('en_US', future_option='bar', another_option=42) collation = Collation( 'en_US', caseLevel=True, caseFirst=CollationCaseFirst.UPPER, strength=CollationStrength.QUATERNARY, numericOrdering=True, alternate=CollationAlternate.SHIFTED, maxVariable=CollationMaxVariable.SPACE, normalization=True, backwards=True) self.assertEqual({ 'locale': 'en_US', 'caseLevel': True, 'caseFirst': 'upper', 'strength': 4, 'numericOrdering': True, 'alternate': 'shifted', 'maxVariable': 'space', 'normalization': True, 'backwards': True }, collation.document) self.assertEqual({ 'locale': 'en_US', 'backwards': True }, Collation('en_US', backwards=True).document) def raisesConfigurationErrorForOldMongoDB(func): @functools.wraps(func) def wrapper(self, *args, **kwargs): if client_context.version.at_least(3, 3, 9): return func(self, *args, **kwargs) else: with self.assertRaises(ConfigurationError): return func(self, *args, **kwargs) return wrapper class TestCollation(unittest.TestCase): @classmethod @client_context.require_connection def setUpClass(cls): cls.listener = EventListener() cls.saved_listeners = monitoring._LISTENERS monitoring._LISTENERS = monitoring._Listeners([], [], [], []) cls.client = rs_or_single_client(event_listeners=[cls.listener]) cls.db = cls.client.pymongo_test cls.collation = Collation('en_US') cls.warn_context = warnings.catch_warnings() cls.warn_context.__enter__() warnings.simplefilter("ignore", DeprecationWarning) @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners cls.warn_context.__exit__() cls.warn_context = None def tearDown(self): self.listener.results.clear() def last_command_started(self): return self.listener.results['started'][-1].command def assertCollationInLastCommand(self): self.assertEqual( self.collation.document, self.last_command_started()['collation']) @raisesConfigurationErrorForOldMongoDB def test_create_collection(self): self.db.test.drop() self.db.create_collection('test', collation=self.collation) self.assertCollationInLastCommand() # Test passing collation as a dict as well. self.db.test.drop() self.listener.results.clear() self.db.create_collection('test', collation=self.collation.document) self.assertCollationInLastCommand() def test_index_model(self): model = IndexModel([('a', 1), ('b', -1)], collation=self.collation) self.assertEqual(self.collation.document, model.document['collation']) @raisesConfigurationErrorForOldMongoDB def test_create_index(self): self.db.test.create_index('foo', collation=self.collation) ci_cmd = self.listener.results['started'][0].command self.assertEqual( self.collation.document, ci_cmd['indexes'][0]['collation']) @raisesConfigurationErrorForOldMongoDB def test_ensure_index(self): self.db.test.ensure_index('foo', collation=self.collation) ci_cmd = self.listener.results['started'][0].command self.assertEqual( self.collation.document, ci_cmd['indexes'][0]['collation']) @raisesConfigurationErrorForOldMongoDB def test_aggregate(self): self.db.test.aggregate([{'$group': {'_id': 42}}], collation=self.collation) self.assertCollationInLastCommand() @raisesConfigurationErrorForOldMongoDB def test_count(self): self.db.test.count(collation=self.collation) self.assertCollationInLastCommand() self.listener.results.clear() self.db.test.find(collation=self.collation).count() self.assertCollationInLastCommand() @raisesConfigurationErrorForOldMongoDB def test_distinct(self): self.db.test.distinct('foo', collation=self.collation) self.assertCollationInLastCommand() self.listener.results.clear() self.db.test.find(collation=self.collation).distinct('foo') self.assertCollationInLastCommand() @raisesConfigurationErrorForOldMongoDB def test_find_command(self): self.db.test.insert_one({'is this thing on?': True}) self.listener.results.clear() next(self.db.test.find(collation=self.collation)) self.assertCollationInLastCommand() @raisesConfigurationErrorForOldMongoDB def test_explain_command(self): self.listener.results.clear() self.db.test.find(collation=self.collation).explain() # The collation should be part of the explained command. self.assertEqual( self.collation.document, self.last_command_started()['explain']['collation']) @raisesConfigurationErrorForOldMongoDB def test_group(self): self.db.test.group('foo', {'foo': {'$gt': 42}}, {}, 'function(a, b) { return a; }', collation=self.collation) self.assertCollationInLastCommand() @raisesConfigurationErrorForOldMongoDB def test_map_reduce(self): self.db.test.map_reduce('function() {}', 'function() {}', 'output', collation=self.collation) self.assertCollationInLastCommand() @raisesConfigurationErrorForOldMongoDB def test_delete(self): self.db.test.delete_one({'foo': 42}, collation=self.collation) command = self.listener.results['started'][0].command self.assertEqual( self.collation.document, command['deletes'][0]['collation']) self.listener.results.clear() self.db.test.delete_many({'foo': 42}, collation=self.collation) command = self.listener.results['started'][0].command self.assertEqual( self.collation.document, command['deletes'][0]['collation']) self.listener.results.clear() self.db.test.remove({'foo': 42}, collation=self.collation) command = self.listener.results['started'][0].command self.assertEqual( self.collation.document, command['deletes'][0]['collation']) @raisesConfigurationErrorForOldMongoDB def test_update(self): self.db.test.update({'foo': 42}, {'$set': {'foo': 'bar'}}, collation=self.collation) command = self.listener.results['started'][0].command self.assertEqual( self.collation.document, command['updates'][0]['collation']) self.listener.results.clear() self.db.test.save({'_id': 12345}, collation=self.collation) command = self.listener.results['started'][0].command self.assertEqual( self.collation.document, command['updates'][0]['collation']) self.listener.results.clear() self.db.test.replace_one({'foo': 42}, {'foo': 43}, collation=self.collation) command = self.listener.results['started'][0].command self.assertEqual( self.collation.document, command['updates'][0]['collation']) self.listener.results.clear() self.db.test.update_one({'foo': 42}, {'$set': {'foo': 43}}, collation=self.collation) command = self.listener.results['started'][0].command self.assertEqual( self.collation.document, command['updates'][0]['collation']) self.listener.results.clear() self.db.test.update_many({'foo': 42}, {'$set': {'foo': 43}}, collation=self.collation) command = self.listener.results['started'][0].command self.assertEqual( self.collation.document, command['updates'][0]['collation']) @raisesConfigurationErrorForOldMongoDB def test_find_and(self): self.db.test.find_and_modify({'foo': 42}, {'$set': {'foo': 43}}, collation=self.collation) self.assertCollationInLastCommand() self.listener.results.clear() self.db.test.find_one_and_delete({'foo': 42}, collation=self.collation) self.assertCollationInLastCommand() self.listener.results.clear() self.db.test.find_one_and_update({'foo': 42}, {'$set': {'foo': 43}}, collation=self.collation) self.assertCollationInLastCommand() self.listener.results.clear() self.db.test.find_one_and_replace({'foo': 42}, {'foo': 43}, collation=self.collation) self.assertCollationInLastCommand() @raisesConfigurationErrorForOldMongoDB def test_bulk_write(self): self.db.test.collection.bulk_write([ DeleteOne({'noCollation': 42}), DeleteMany({'noCollation': 42}), DeleteOne({'foo': 42}, collation=self.collation), DeleteMany({'foo': 42}, collation=self.collation), ReplaceOne({'noCollation': 24}, {'bar': 42}), UpdateOne({'noCollation': 84}, {'$set': {'bar': 10}}, upsert=True), UpdateMany({'noCollation': 45}, {'$set': {'bar': 42}}), ReplaceOne({'foo': 24}, {'foo': 42}, collation=self.collation), UpdateOne({'foo': 84}, {'$set': {'foo': 10}}, upsert=True, collation=self.collation), UpdateMany({'foo': 45}, {'$set': {'foo': 42}}, collation=self.collation) ]) delete_cmd = self.listener.results['started'][0].command update_cmd = self.listener.results['started'][1].command def check_ops(ops): for op in ops: if 'noCollation' in op['q']: self.assertNotIn('collation', op) else: self.assertEqual(self.collation.document, op['collation']) check_ops(delete_cmd['deletes']) check_ops(update_cmd['updates']) @raisesConfigurationErrorForOldMongoDB def test_bulk(self): bulk = self.db.test.initialize_ordered_bulk_op() bulk.find({'noCollation': 42}).remove_one() bulk.find({'noCollation': 42}).remove() bulk.find({'foo': 42}, collation=self.collation).remove_one() bulk.find({'foo': 42}, collation=self.collation).remove() bulk.find({'noCollation': 24}).replace_one({'bar': 42}) bulk.find({'noCollation': 84}).upsert().update_one( {'$set': {'foo': 10}}) bulk.find({'noCollation': 45}).update({'$set': {'bar': 42}}) bulk.find({'foo': 24}, collation=self.collation).replace_one( {'foo': 42}) bulk.find({'foo': 84}, collation=self.collation).upsert().update_one( {'$set': {'foo': 10}}) bulk.find({'foo': 45}, collation=self.collation).update({ '$set': {'foo': 42}}) bulk.execute() delete_cmd = self.listener.results['started'][0].command update_cmd = self.listener.results['started'][1].command def check_ops(ops): for op in ops: if 'noCollation' in op['q']: self.assertNotIn('collation', op) else: self.assertEqual(self.collation.document, op['collation']) check_ops(delete_cmd['deletes']) check_ops(update_cmd['updates']) @client_context.require_version_max(3, 3, 8) def test_mixed_bulk_collation(self): bulk = self.db.test.initialize_unordered_bulk_op() bulk.find({'foo': 42}).upsert().update_one( {'$set': {'bar': 10}}) bulk.find({'foo': 43}, collation=self.collation).remove_one() with self.assertRaises(ConfigurationError): bulk.execute() self.assertIsNone(self.db.test.find_one({'foo': 42})) @raisesConfigurationErrorForOldMongoDB def test_indexes_same_keys_different_collations(self): self.db.test.drop() usa_collation = Collation('en_US') ja_collation = Collation('ja') self.db.test.create_indexes([ IndexModel('fieldname', collation=usa_collation), IndexModel('fieldname', name='japanese_version', collation=ja_collation), IndexModel('fieldname', name='simple') ]) indexes = self.db.test.index_information() self.assertEqual(usa_collation.document['locale'], indexes['fieldname_1']['collation']['locale']) self.assertEqual(ja_collation.document['locale'], indexes['japanese_version']['collation']['locale']) self.assertNotIn('collation', indexes['simple']) self.db.test.drop_index('fieldname_1') indexes = self.db.test.index_information() self.assertIn('japanese_version', indexes) self.assertIn('simple', indexes) self.assertNotIn('fieldname', indexes) def test_unacknowledged_write(self): unacknowledged = WriteConcern(w=0) collection = self.db.get_collection( 'test', write_concern=unacknowledged) with self.assertRaises(ConfigurationError): collection.update_one( {'hello': 'world'}, {'$set': {'hello': 'moon'}}, collation=self.collation) bulk = collection.initialize_ordered_bulk_op() bulk.find({'hello': 'world'}, collation=self.collation).update_one( {'$set': {'hello': 'moon'}}) with self.assertRaises(ConfigurationError): bulk.execute() update_one = UpdateOne({'hello': 'world'}, {'$set': {'hello': 'moon'}}, collation=self.collation) with self.assertRaises(ConfigurationError): collection.bulk_write([update_one]) @raisesConfigurationErrorForOldMongoDB def test_cursor_collation(self): self.db.test.insert_one({'hello': 'world'}) next(self.db.test.find().collation(self.collation)) self.assertCollationInLastCommand() pymongo-3.6.1/test/test_client_context.py0000644000076600000240000000224013245621354021050 0ustar shanestaff00000000000000# Copyright 2018-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys sys.path[0:0] = [""] from test import client_context, SkipTest, unittest class TestClientContext(unittest.TestCase): def test_must_connect(self): if 'PYMONGO_MUST_CONNECT' not in os.environ: raise SkipTest('PYMONGO_MUST_CONNECT is not set') self.assertTrue(client_context.connected, 'client context must be connected when ' 'PYMONGO_MUST_CONNECT is set. Failed attempts:\n%s' % (client_context.connection_attempt_info(),)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_examples.py0000644000076600000240000005537413245621354017664 0ustar shanestaff00000000000000# Copyright 2017 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """MongoDB documentation examples in Python.""" import threading import sys sys.path[0:0] = [""] from test import client_context, unittest class TestSampleShellCommands(unittest.TestCase): @classmethod @client_context.require_connection def setUpClass(cls): # Run once before any tests run. client_context.client.pymongo_test.inventory.drop() def tearDown(self): # Run after every test. client_context.client.pymongo_test.inventory.drop() def test_first_three_examples(self): db = client_context.client.pymongo_test # Start Example 1 db.inventory.insert_one( {"item": "canvas", "qty": 100, "tags": ["cotton"], "size": {"h": 28, "w": 35.5, "uom": "cm"}}) # End Example 1 self.assertEqual(db.inventory.count(), 1) # Start Example 2 cursor = db.inventory.find({"item": "canvas"}) # End Example 2 self.assertEqual(cursor.count(), 1) # Start Example 3 db.inventory.insert_many([ {"item": "journal", "qty": 25, "tags": ["blank", "red"], "size": {"h": 14, "w": 21, "uom": "cm"}}, {"item": "mat", "qty": 85, "tags": ["gray"], "size": {"h": 27.9, "w": 35.5, "uom": "cm"}}, {"item": "mousepad", "qty": 25, "tags": ["gel", "blue"], "size": {"h": 19, "w": 22.85, "uom": "cm"}}]) # End Example 3 self.assertEqual(db.inventory.count(), 4) def test_query_top_level_fields(self): db = client_context.client.pymongo_test # Start Example 6 db.inventory.insert_many([ {"item": "journal", "qty": 25, "size": {"h": 14, "w": 21, "uom": "cm"}, "status": "A"}, {"item": "notebook", "qty": 50, "size": {"h": 8.5, "w": 11, "uom": "in"}, "status": "A"}, {"item": "paper", "qty": 100, "size": {"h": 8.5, "w": 11, "uom": "in"}, "status": "D"}, {"item": "planner", "qty": 75, "size": {"h": 22.85, "w": 30, "uom": "cm"}, "status": "D"}, {"item": "postcard", "qty": 45, "size": {"h": 10, "w": 15.25, "uom": "cm"}, "status": "A"}]) # End Example 6 self.assertEqual(db.inventory.count(), 5) # Start Example 7 cursor = db.inventory.find({}) # End Example 7 self.assertEqual(len(list(cursor)), 5) # Start Example 9 cursor = db.inventory.find({"status": "D"}) # End Example 9 self.assertEqual(len(list(cursor)), 2) # Start Example 10 cursor = db.inventory.find({"status": {"$in": ["A", "D"]}}) # End Example 10 self.assertEqual(len(list(cursor)), 5) # Start Example 11 cursor = db.inventory.find({"status": "A", "qty": {"$lt": 30}}) # End Example 11 self.assertEqual(len(list(cursor)), 1) # Start Example 12 cursor = db.inventory.find( {"$or": [{"status": "A"}, {"qty": {"$lt": 30}}]}) # End Example 12 self.assertEqual(len(list(cursor)), 3) # Start Example 13 cursor = db.inventory.find({ "status": "A", "$or": [{"qty": {"$lt": 30}}, {"item": {"$regex": "^p"}}]}) # End Example 13 self.assertEqual(len(list(cursor)), 2) def test_query_embedded_documents(self): db = client_context.client.pymongo_test # Start Example 14 # Subdocument key order matters in a few of these examples so we have # to use bson.son.SON instead of a Python dict. from bson.son import SON db.inventory.insert_many([ {"item": "journal", "qty": 25, "size": SON([("h", 14), ("w", 21), ("uom", "cm")]), "status": "A"}, {"item": "notebook", "qty": 50, "size": SON([("h", 8.5), ("w", 11), ("uom", "in")]), "status": "A"}, {"item": "paper", "qty": 100, "size": SON([("h", 8.5), ("w", 11), ("uom", "in")]), "status": "D"}, {"item": "planner", "qty": 75, "size": SON([("h", 22.85), ("w", 30), ("uom", "cm")]), "status": "D"}, {"item": "postcard", "qty": 45, "size": SON([("h", 10), ("w", 15.25), ("uom", "cm")]), "status": "A"}]) # End Example 14 # Start Example 15 cursor = db.inventory.find( {"size": SON([("h", 14), ("w", 21), ("uom", "cm")])}) # End Example 15 self.assertEqual(len(list(cursor)), 1) # Start Example 16 cursor = db.inventory.find( {"size": SON([("w", 21), ("h", 14), ("uom", "cm")])}) # End Example 16 self.assertEqual(len(list(cursor)), 0) # Start Example 17 cursor = db.inventory.find({"size.uom": "in"}) # End Example 17 self.assertEqual(len(list(cursor)), 2) # Start Example 18 cursor = db.inventory.find({"size.h": {"$lt": 15}}) # End Example 18 self.assertEqual(len(list(cursor)), 4) # Start Example 19 cursor = db.inventory.find( {"size.h": {"$lt": 15}, "size.uom": "in", "status": "D"}) # End Example 19 self.assertEqual(len(list(cursor)), 1) def test_query_arrays(self): db = client_context.client.pymongo_test # Start Example 20 db.inventory.insert_many([ {"item": "journal", "qty": 25, "tags": ["blank", "red"], "dim_cm": [14, 21]}, {"item": "notebook", "qty": 50, "tags": ["red", "blank"], "dim_cm": [14, 21]}, {"item": "paper", "qty": 100, "tags": ["red", "blank", "plain"], "dim_cm": [14, 21]}, {"item": "planner", "qty": 75, "tags": ["blank", "red"], "dim_cm": [22.85, 30]}, {"item": "postcard", "qty": 45, "tags": ["blue"], "dim_cm": [10, 15.25]}]) # End Example 20 # Start Example 21 cursor = db.inventory.find({"tags": ["red", "blank"]}) # End Example 21 self.assertEqual(len(list(cursor)), 1) # Start Example 22 cursor = db.inventory.find({"tags": {"$all": ["red", "blank"]}}) # End Example 22 self.assertEqual(len(list(cursor)), 4) # Start Example 23 cursor = db.inventory.find({"tags": "red"}) # End Example 23 self.assertEqual(len(list(cursor)), 4) # Start Example 24 cursor = db.inventory.find({"dim_cm": {"$gt": 25}}) # End Example 24 self.assertEqual(len(list(cursor)), 1) # Start Example 25 cursor = db.inventory.find({"dim_cm": {"$gt": 15, "$lt": 20}}) # End Example 25 self.assertEqual(len(list(cursor)), 4) # Start Example 26 cursor = db.inventory.find( {"dim_cm": {"$elemMatch": {"$gt": 22, "$lt": 30}}}) # End Example 26 self.assertEqual(len(list(cursor)), 1) # Start Example 27 cursor = db.inventory.find({"dim_cm.1": {"$gt": 25}}) # End Example 27 self.assertEqual(len(list(cursor)), 1) # Start Example 28 cursor = db.inventory.find({"tags": {"$size": 3}}) # End Example 28 self.assertEqual(len(list(cursor)), 1) def test_query_array_of_documents(self): db = client_context.client.pymongo_test # Start Example 29 # Subdocument key order matters in a few of these examples so we have # to use bson.son.SON instead of a Python dict. from bson.son import SON db.inventory.insert_many([ {"item": "journal", "instock": [ SON([("warehouse", "A"), ("qty", 5)]), SON([("warehouse", "C"), ("qty", 15)])]}, {"item": "notebook", "instock": [ SON([("warehouse", "C"), ("qty", 5)])]}, {"item": "paper", "instock": [ SON([("warehouse", "A"), ("qty", 60)]), SON([("warehouse", "B"), ("qty", 15)])]}, {"item": "planner", "instock": [ SON([("warehouse", "A"), ("qty", 40)]), SON([("warehouse", "B"), ("qty", 5)])]}, {"item": "postcard", "instock": [ SON([("warehouse", "B"), ("qty", 15)]), SON([("warehouse", "C"), ("qty", 35)])]}]) # End Example 29 # Start Example 30 cursor = db.inventory.find( {"instock": SON([("warehouse", "A"), ("qty", 5)])}) # End Example 30 self.assertEqual(len(list(cursor)), 1) # Start Example 31 cursor = db.inventory.find( {"instock": SON([("qty", 5), ("warehouse", "A")])}) # End Example 31 self.assertEqual(len(list(cursor)), 0) # Start Example 32 cursor = db.inventory.find({'instock.0.qty': {"$lte": 20}}) # End Example 32 self.assertEqual(len(list(cursor)), 3) # Start Example 33 cursor = db.inventory.find({'instock.qty': {"$lte": 20}}) # End Example 33 self.assertEqual(len(list(cursor)), 5) # Start Example 34 cursor = db.inventory.find( {"instock": {"$elemMatch": {"qty": 5, "warehouse": "A"}}}) # End Example 34 self.assertEqual(len(list(cursor)), 1) # Start Example 35 cursor = db.inventory.find( {"instock": {"$elemMatch": {"qty": {"$gt": 10, "$lte": 20}}}}) # End Example 35 self.assertEqual(len(list(cursor)), 3) # Start Example 36 cursor = db.inventory.find({"instock.qty": {"$gt": 10, "$lte": 20}}) # End Example 36 self.assertEqual(len(list(cursor)), 4) # Start Example 37 cursor = db.inventory.find( {"instock.qty": 5, "instock.warehouse": "A"}) # End Example 37 self.assertEqual(len(list(cursor)), 2) def test_query_null(self): db = client_context.client.pymongo_test # Start Example 38 db.inventory.insert_many([{"_id": 1, "item": None}, {"_id": 2}]) # End Example 38 # Start Example 39 cursor = db.inventory.find({"item": None}) # End Example 39 self.assertEqual(len(list(cursor)), 2) # Start Example 40 cursor = db.inventory.find({"item": {"$type": 10}}) # End Example 40 self.assertEqual(len(list(cursor)), 1) # Start Example 41 cursor = db.inventory.find({"item": {"$exists": False}}) # End Example 41 self.assertEqual(len(list(cursor)), 1) def test_projection(self): db = client_context.client.pymongo_test # Start Example 42 db.inventory.insert_many([ {"item": "journal", "status": "A", "size": {"h": 14, "w": 21, "uom": "cm"}, "instock": [{"warehouse": "A", "qty": 5}]}, {"item": "notebook", "status": "A", "size": {"h": 8.5, "w": 11, "uom": "in"}, "instock": [{"warehouse": "C", "qty": 5}]}, {"item": "paper", "status": "D", "size": {"h": 8.5, "w": 11, "uom": "in"}, "instock": [{"warehouse": "A", "qty": 60}]}, {"item": "planner", "status": "D", "size": {"h": 22.85, "w": 30, "uom": "cm"}, "instock": [{"warehouse": "A", "qty": 40}]}, {"item": "postcard", "status": "A", "size": {"h": 10, "w": 15.25, "uom": "cm"}, "instock": [ {"warehouse": "B", "qty": 15}, {"warehouse": "C", "qty": 35}]}]) # End Example 42 # Start Example 43 cursor = db.inventory.find({"status": "A"}) # End Example 43 self.assertEqual(len(list(cursor)), 3) # Start Example 44 cursor = db.inventory.find( {"status": "A"}, {"item": 1, "status": 1}) # End Example 44 for doc in cursor: self.assertTrue("_id" in doc) self.assertTrue("item" in doc) self.assertTrue("status" in doc) self.assertFalse("size" in doc) self.assertFalse("instock" in doc) # Start Example 45 cursor = db.inventory.find( {"status": "A"}, {"item": 1, "status": 1, "_id": 0}) # End Example 45 for doc in cursor: self.assertFalse("_id" in doc) self.assertTrue("item" in doc) self.assertTrue("status" in doc) self.assertFalse("size" in doc) self.assertFalse("instock" in doc) # Start Example 46 cursor = db.inventory.find( {"status": "A"}, {"status": 0, "instock": 0}) # End Example 46 for doc in cursor: self.assertTrue("_id" in doc) self.assertTrue("item" in doc) self.assertFalse("status" in doc) self.assertTrue("size" in doc) self.assertFalse("instock" in doc) # Start Example 47 cursor = db.inventory.find( {"status": "A"}, {"item": 1, "status": 1, "size.uom": 1}) # End Example 47 for doc in cursor: self.assertTrue("_id" in doc) self.assertTrue("item" in doc) self.assertTrue("status" in doc) self.assertTrue("size" in doc) self.assertFalse("instock" in doc) size = doc['size'] self.assertTrue('uom' in size) self.assertFalse('h' in size) self.assertFalse('w' in size) # Start Example 48 cursor = db.inventory.find({"status": "A"}, {"size.uom": 0}) # End Example 48 for doc in cursor: self.assertTrue("_id" in doc) self.assertTrue("item" in doc) self.assertTrue("status" in doc) self.assertTrue("size" in doc) self.assertTrue("instock" in doc) size = doc['size'] self.assertFalse('uom' in size) self.assertTrue('h' in size) self.assertTrue('w' in size) # Start Example 49 cursor = db.inventory.find( {"status": "A"}, {"item": 1, "status": 1, "instock.qty": 1}) # End Example 49 for doc in cursor: self.assertTrue("_id" in doc) self.assertTrue("item" in doc) self.assertTrue("status" in doc) self.assertFalse("size" in doc) self.assertTrue("instock" in doc) for subdoc in doc['instock']: self.assertFalse('warehouse' in subdoc) self.assertTrue('qty' in subdoc) # Start Example 50 cursor = db.inventory.find( {"status": "A"}, {"item": 1, "status": 1, "instock": {"$slice": -1}}) # End Example 50 for doc in cursor: self.assertTrue("_id" in doc) self.assertTrue("item" in doc) self.assertTrue("status" in doc) self.assertFalse("size" in doc) self.assertTrue("instock" in doc) self.assertEqual(len(doc["instock"]), 1) def test_update_and_replace(self): db = client_context.client.pymongo_test # Start Example 51 db.inventory.insert_many([ {"item": "canvas", "qty": 100, "size": {"h": 28, "w": 35.5, "uom": "cm"}, "status": "A"}, {"item": "journal", "qty": 25, "size": {"h": 14, "w": 21, "uom": "cm"}, "status": "A"}, {"item": "mat", "qty": 85, "size": {"h": 27.9, "w": 35.5, "uom": "cm"}, "status": "A"}, {"item": "mousepad", "qty": 25, "size": {"h": 19, "w": 22.85, "uom": "cm"}, "status": "P"}, {"item": "notebook", "qty": 50, "size": {"h": 8.5, "w": 11, "uom": "in"}, "status": "P"}, {"item": "paper", "qty": 100, "size": {"h": 8.5, "w": 11, "uom": "in"}, "status": "D"}, {"item": "planner", "qty": 75, "size": {"h": 22.85, "w": 30, "uom": "cm"}, "status": "D"}, {"item": "postcard", "qty": 45, "size": {"h": 10, "w": 15.25, "uom": "cm"}, "status": "A"}, {"item": "sketchbook", "qty": 80, "size": {"h": 14, "w": 21, "uom": "cm"}, "status": "A"}, {"item": "sketch pad", "qty": 95, "size": {"h": 22.85, "w": 30.5, "uom": "cm"}, "status": "A"}]) # End Example 51 # Start Example 52 db.inventory.update_one( {"item": "paper"}, {"$set": {"size.uom": "cm", "status": "P"}, "$currentDate": {"lastModified": True}}) # End Example 52 for doc in db.inventory.find({"item": "paper"}): self.assertEqual(doc["size"]["uom"], "cm") self.assertEqual(doc["status"], "P") self.assertTrue("lastModified" in doc) # Start Example 53 db.inventory.update_many( {"qty": {"$lt": 50}}, {"$set": {"size.uom": "in", "status": "P"}, "$currentDate": {"lastModified": True}}) # End Example 53 for doc in db.inventory.find({"qty": {"$lt": 50}}): self.assertEqual(doc["size"]["uom"], "in") self.assertEqual(doc["status"], "P") self.assertTrue("lastModified" in doc) # Start Example 54 db.inventory.replace_one( {"item": "paper"}, {"item": "paper", "instock": [ {"warehouse": "A", "qty": 60}, {"warehouse": "B", "qty": 40}]}) # End Example 54 for doc in db.inventory.find({"item": "paper"}, {"_id": 0}): self.assertEqual(len(doc.keys()), 2) self.assertTrue("item" in doc) self.assertTrue("instock" in doc) self.assertEqual(len(doc["instock"]), 2) def test_delete(self): db = client_context.client.pymongo_test # Start Example 55 db.inventory.insert_many([ {"item": "journal", "qty": 25, "size": {"h": 14, "w": 21, "uom": "cm"}, "status": "A"}, {"item": "notebook", "qty": 50, "size": {"h": 8.5, "w": 11, "uom": "in"}, "status": "P"}, {"item": "paper", "qty": 100, "size": {"h": 8.5, "w": 11, "uom": "in"}, "status": "D"}, {"item": "planner", "qty": 75, "size": {"h": 22.85, "w": 30, "uom": "cm"}, "status": "D"}, {"item": "postcard", "qty": 45, "size": {"h": 10, "w": 15.25, "uom": "cm"}, "status": "A"}]) # End Example 55 self.assertEqual(db.inventory.count(), 5) # Start Example 57 db.inventory.delete_many({"status": "A"}) # End Example 57 self.assertEqual(db.inventory.count(), 3) # Start Example 58 db.inventory.delete_one({"status": "D"}) # End Example 58 self.assertEqual(db.inventory.count(), 2) # Start Example 56 db.inventory.delete_many({}) # End Example 56 self.assertEqual(db.inventory.count(), 0) @client_context.require_version_min(3, 5, 11) @client_context.require_replica_set def test_change_streams(self): db = client_context.client.pymongo_test done = False def insert_docs(): while not done: db.inventory.insert_one({"username": "alice"}) db.inventory.delete_one({"username": "alice"}) t = threading.Thread(target=insert_docs) t.start() try: # 1. The database for reactive, real-time applications # Start Changestream Example 1 cursor = db.inventory.watch() document = next(cursor) # End Changestream Example 1 # Start Changestream Example 2 cursor = db.inventory.watch(full_document='updateLookup') document = next(cursor) # End Changestream Example 2 # Start Changestream Example 3 resume_token = document.get("_id") cursor = db.inventory.watch(resume_after=resume_token) document = next(cursor) # End Changestream Example 3 # Start Changestream Example 4 pipeline = [ {"$match": {"$or": [ {"fullDocument.username": "alice"}, {"operationType": {"$in": ["delete"]}}] } } ] cursor = db.inventory.watch(pipeline=pipeline) document = next(cursor) # End Changestream Example 4 finally: done = True t.join() @client_context.require_version_min(3, 6, 0) @client_context.require_replica_set def test_misc(self): # Marketing examples client = client_context.client self.addCleanup(client.drop_database, "test") self.addCleanup(client.drop_database, "my_database") # 2. Tunable consistency controls collection = client.my_database.my_collection with client.start_session() as session: collection.insert_one({'_id': 1}, session=session) collection.update_one( {'_id': 1}, {"$set": {"a": 1}}, session=session) for doc in collection.find({}, session=session): pass # 3. Exploiting the power of arrays collection = client.test.array_updates_test collection.update_one( {'_id': 1}, {"$set": {"a.$[i].b": 2}}, array_filters=[{"i.b": 0}]) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_dbref.py0000644000076600000240000001230613245621354017114 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the dbref module.""" import pickle import sys sys.path[0:0] = [""] from bson.dbref import DBRef from bson.objectid import ObjectId from test import unittest from copy import deepcopy class TestDBRef(unittest.TestCase): def test_creation(self): a = ObjectId() self.assertRaises(TypeError, DBRef) self.assertRaises(TypeError, DBRef, "coll") self.assertRaises(TypeError, DBRef, 4, a) self.assertRaises(TypeError, DBRef, 1.5, a) self.assertRaises(TypeError, DBRef, a, a) self.assertRaises(TypeError, DBRef, None, a) self.assertRaises(TypeError, DBRef, "coll", a, 5) self.assertTrue(DBRef("coll", a)) self.assertTrue(DBRef(u"coll", a)) self.assertTrue(DBRef(u"coll", 5)) self.assertTrue(DBRef(u"coll", 5, "database")) def test_read_only(self): a = DBRef("coll", ObjectId()) def foo(): a.collection = "blah" def bar(): a.id = "aoeu" self.assertEqual("coll", a.collection) a.id self.assertEqual(None, a.database) self.assertRaises(AttributeError, foo) self.assertRaises(AttributeError, bar) def test_repr(self): self.assertEqual(repr(DBRef("coll", ObjectId("1234567890abcdef12345678"))), "DBRef('coll', ObjectId('1234567890abcdef12345678'))") self.assertEqual(repr(DBRef(u"coll", ObjectId("1234567890abcdef12345678"))), "DBRef(%s, ObjectId('1234567890abcdef12345678'))" % (repr(u'coll'),) ) self.assertEqual(repr(DBRef("coll", 5, foo="bar")), "DBRef('coll', 5, foo='bar')") self.assertEqual(repr(DBRef("coll", ObjectId("1234567890abcdef12345678"), "foo")), "DBRef('coll', ObjectId('1234567890abcdef12345678'), " "'foo')") def test_equality(self): obj_id = ObjectId("1234567890abcdef12345678") self.assertEqual(DBRef('foo', 5), DBRef('foo', 5)) self.assertEqual(DBRef("coll", obj_id), DBRef(u"coll", obj_id)) self.assertNotEqual(DBRef("coll", obj_id), DBRef(u"coll", obj_id, "foo")) self.assertNotEqual(DBRef("coll", obj_id), DBRef("col", obj_id)) self.assertNotEqual(DBRef("coll", obj_id), DBRef("coll", ObjectId(b"123456789011"))) self.assertNotEqual(DBRef("coll", obj_id), 4) self.assertEqual(DBRef("coll", obj_id, "foo"), DBRef(u"coll", obj_id, "foo")) self.assertNotEqual(DBRef("coll", obj_id, "foo"), DBRef(u"coll", obj_id, "bar")) # Explicitly test inequality self.assertFalse(DBRef('foo', 5) != DBRef('foo', 5)) self.assertFalse(DBRef("coll", obj_id) != DBRef(u"coll", obj_id)) self.assertFalse(DBRef("coll", obj_id, "foo") != DBRef(u"coll", obj_id, "foo")) def test_kwargs(self): self.assertEqual(DBRef("coll", 5, foo="bar"), DBRef("coll", 5, foo="bar")) self.assertNotEqual(DBRef("coll", 5, foo="bar"), DBRef("coll", 5)) self.assertNotEqual(DBRef("coll", 5, foo="bar"), DBRef("coll", 5, foo="baz")) self.assertEqual("bar", DBRef("coll", 5, foo="bar").foo) self.assertRaises(AttributeError, getattr, DBRef("coll", 5, foo="bar"), "bar") def test_deepcopy(self): a = DBRef('coll', 'asdf', 'db', x=[1]) b = deepcopy(a) self.assertEqual(a, b) self.assertNotEqual(id(a), id(b.x)) self.assertEqual(a.x, b.x) self.assertNotEqual(id(a.x), id(b.x)) b.x[0] = 2 self.assertEqual(a.x, [1]) self.assertEqual(b.x, [2]) def test_pickling(self): dbr = DBRef('coll', 5, foo='bar') for protocol in [0, 1, 2, -1]: pkl = pickle.dumps(dbr, protocol=protocol) dbr2 = pickle.loads(pkl) self.assertEqual(dbr, dbr2) def test_dbref_hash(self): dbref_1a = DBRef('collection', 'id', 'database') dbref_1b = DBRef('collection', 'id', 'database') self.assertEqual(hash(dbref_1a), hash(dbref_1b)) dbref_2a = DBRef('collection', 'id', 'database', custom='custom') dbref_2b = DBRef('collection', 'id', 'database', custom='custom') self.assertEqual(hash(dbref_2a), hash(dbref_2b)) self.assertNotEqual(hash(dbref_1a), hash(dbref_2a)) if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_decimal128.py0000644000076600000240000000553513245621354017671 0ustar shanestaff00000000000000# Copyright 2016-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for Decimal128.""" import codecs import glob import json import os.path import pickle import sys from binascii import unhexlify from decimal import Decimal, DecimalException sys.path[0:0] = [""] from bson import BSON from bson.decimal128 import Decimal128, create_decimal128_context from bson.json_util import dumps, loads from bson.py3compat import b from test import client_context, unittest class TestDecimal128(unittest.TestCase): def test_round_trip(self): if not client_context.version.at_least(3, 3, 6): raise unittest.SkipTest( 'Round trip test requires MongoDB >= 3.3.6') coll = client_context.client.pymongo_test.test coll.drop() dec128 = Decimal128.from_bid( b'\x00@cR\xbf\xc6\x01\x00\x00\x00\x00\x00\x00\x00\x1c0') coll.insert_one({'dec128': dec128}) doc = coll.find_one({'dec128': dec128}) self.assertIsNotNone(doc) self.assertEqual(doc['dec128'], dec128) def test_pickle(self): dec128 = Decimal128.from_bid( b'\x00@cR\xbf\xc6\x01\x00\x00\x00\x00\x00\x00\x00\x1c0') for protocol in range(pickle.HIGHEST_PROTOCOL + 1): pkl = pickle.dumps(dec128, protocol=protocol) self.assertEqual(dec128, pickle.loads(pkl)) def test_special(self): dnan = Decimal('NaN') dnnan = Decimal('-NaN') dsnan = Decimal('sNaN') dnsnan = Decimal('-sNaN') dnan128 = Decimal128(dnan) dnnan128 = Decimal128(dnnan) dsnan128 = Decimal128(dsnan) dnsnan128 = Decimal128(dnsnan) # Due to the comparison rules for decimal.Decimal we have to # compare strings. self.assertEqual(str(dnan), str(dnan128.to_decimal())) self.assertEqual(str(dnnan), str(dnnan128.to_decimal())) self.assertEqual(str(dsnan), str(dsnan128.to_decimal())) self.assertEqual(str(dnsnan), str(dnsnan128.to_decimal())) def test_decimal128_context(self): ctx = create_decimal128_context() self.assertEqual("NaN", str(ctx.copy().create_decimal(".13.1"))) self.assertEqual("Infinity", str(ctx.copy().create_decimal("1E6145"))) self.assertEqual("0E-6176", str(ctx.copy().create_decimal("1E-6177"))) if __name__ == '__main__': unittest.main() pymongo-3.6.1/test/mod_wsgi_test/0000755000076600000240000000000013246104133017256 5ustar shanestaff00000000000000pymongo-3.6.1/test/mod_wsgi_test/test_client.py0000644000076600000240000001132213245621354022154 0ustar shanestaff00000000000000# Copyright 2012-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test client for mod_wsgi application, see bug PYTHON-353. """ import sys import threading import time from optparse import OptionParser try: from urllib2 import urlopen except ImportError: # Python 3. from urllib.request import urlopen try: import thread except ImportError: # Python 3. import _thread as thread def parse_args(): parser = OptionParser("""usage: %prog [options] mode url mode:\tparallel or serial""") # Should be enough that any connection leak will exhaust available file # descriptors. parser.add_option( "-n", "--nrequests", type="int", dest="nrequests", default=50 * 1000, help="Number of times to GET the URL, in total") parser.add_option( "-t", "--nthreads", type="int", dest="nthreads", default=100, help="Number of threads with mode 'parallel'") parser.add_option( "-q", "--quiet", action="store_false", dest="verbose", default=True, help="Don't print status messages to stdout") parser.add_option( "-c", "--continue", action="store_true", dest="continue_", default=False, help="Continue after HTTP errors") try: options, (mode, url) = parser.parse_args() except ValueError: parser.print_usage() sys.exit(1) if mode not in ('parallel', 'serial'): parser.print_usage() sys.exit(1) return options, mode, url def get(url): urlopen(url).read().strip() class URLGetterThread(threading.Thread): # Class variables. counter_lock = threading.Lock() counter = 0 def __init__(self, options, url, nrequests_per_thread): super(URLGetterThread, self).__init__() self.options = options self.url = url self.nrequests_per_thread = nrequests_per_thread self.errors = 0 def run(self): for i in range(self.nrequests_per_thread): try: get(url) except Exception as e: print(e) if not options.continue_: thread.interrupt_main() thread.exit() self.errors += 1 with URLGetterThread.counter_lock: URLGetterThread.counter += 1 counter = URLGetterThread.counter should_print = options.verbose and not counter % 1000 if should_print: print(counter) def main(options, mode, url): start_time = time.time() errors = 0 if mode == 'parallel': nrequests_per_thread = options.nrequests // options.nthreads if options.verbose: print ( 'Getting %s %s times total in %s threads, ' '%s times per thread' % ( url, nrequests_per_thread * options.nthreads, options.nthreads, nrequests_per_thread)) threads = [ URLGetterThread(options, url, nrequests_per_thread) for _ in range(options.nthreads) ] for t in threads: t.start() for t in threads: t.join() errors = sum([t.errors for t in threads]) nthreads_with_errors = len([t for t in threads if t.errors]) if nthreads_with_errors: print('%d threads had errors! %d errors in total' % ( nthreads_with_errors, errors)) else: assert mode == 'serial' if options.verbose: print('Getting %s %s times in one thread' % ( url, options.nrequests )) for i in range(1, options.nrequests + 1): try: get(url) except Exception as e: print(e) if not options.continue_: sys.exit(1) errors += 1 if options.verbose and not i % 1000: print(i) if errors: print('%d errors!' % errors) if options.verbose: print('Completed in %.2f seconds' % (time.time() - start_time)) if errors: # Failure sys.exit(1) if __name__ == '__main__': options, mode, url = parse_args() main(options, mode, url) pymongo-3.6.1/test/pymongo_mocks.py0000644000076600000240000001605713245621354017666 0ustar shanestaff00000000000000# Copyright 2013-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for mocking parts of PyMongo to test other parts.""" import contextlib from functools import partial import weakref from pymongo import common from pymongo import MongoClient from pymongo.errors import AutoReconnect, NetworkTimeout from pymongo.ismaster import IsMaster from pymongo.monitor import Monitor from pymongo.pool import Pool from pymongo.server_description import ServerDescription from test import client_context default_host, default_port = client_context.host, client_context.port class MockPool(Pool): def __init__(self, client, pair, *args, **kwargs): # MockPool gets a 'client' arg, regular pools don't. Weakref it to # avoid cycle with __del__, causing ResourceWarnings in Python 3.3. self.client = weakref.proxy(client) self.mock_host, self.mock_port = pair # Actually connect to the default server. Pool.__init__(self, (default_host, default_port), *args, **kwargs) @contextlib.contextmanager def get_socket(self, all_credentials, checkout=False): client = self.client host_and_port = '%s:%s' % (self.mock_host, self.mock_port) if host_and_port in client.mock_down_hosts: raise AutoReconnect('mock error') assert host_and_port in ( client.mock_standalones + client.mock_members + client.mock_mongoses), "bad host: %s" % host_and_port with Pool.get_socket(self, all_credentials) as sock_info: sock_info.mock_host = self.mock_host sock_info.mock_port = self.mock_port yield sock_info class MockMonitor(Monitor): def __init__( self, client, server_description, topology, pool, topology_settings): # MockMonitor gets a 'client' arg, regular monitors don't. self.client = client Monitor.__init__( self, server_description, topology, pool, topology_settings) def _check_once(self, metadata=None, cluster_time=None): address = self._server_description.address response, rtt = self.client.mock_is_master('%s:%d' % address) return ServerDescription(address, IsMaster(response), rtt) class MockClient(MongoClient): def __init__( self, standalones, members, mongoses, ismaster_hosts=None, *args, **kwargs): """A MongoClient connected to the default server, with a mock topology. standalones, members, mongoses determine the configuration of the topology. They are formatted like ['a:1', 'b:2']. ismaster_hosts provides an alternative host list for the server's mocked ismaster response; see test_connect_with_internal_ips. """ self.mock_standalones = standalones[:] self.mock_members = members[:] if self.mock_members: self.mock_primary = self.mock_members[0] else: self.mock_primary = None if ismaster_hosts is not None: self.mock_ismaster_hosts = ismaster_hosts else: self.mock_ismaster_hosts = members[:] self.mock_mongoses = mongoses[:] # Hosts that should raise socket errors. self.mock_down_hosts = [] # Hostname -> (min wire version, max wire version) self.mock_wire_versions = {} # Hostname -> max write batch size self.mock_max_write_batch_sizes = {} # Hostname -> round trip time self.mock_rtts = {} kwargs['_pool_class'] = partial(MockPool, self) kwargs['_monitor_class'] = partial(MockMonitor, self) client_options = client_context.ssl_client_options.copy() client_options.update(kwargs) super(MockClient, self).__init__(*args, **client_options) def kill_host(self, host): """Host is like 'a:1'.""" self.mock_down_hosts.append(host) def revive_host(self, host): """Host is like 'a:1'.""" self.mock_down_hosts.remove(host) def set_wire_version_range(self, host, min_version, max_version): self.mock_wire_versions[host] = (min_version, max_version) def set_max_write_batch_size(self, host, size): self.mock_max_write_batch_sizes[host] = size def mock_is_master(self, host): """Return mock ismaster response (a dict) and round trip time.""" if host in self.mock_wire_versions: min_wire_version, max_wire_version = self.mock_wire_versions[host] else: min_wire_version = common.MIN_SUPPORTED_WIRE_VERSION max_wire_version = common.MAX_SUPPORTED_WIRE_VERSION max_write_batch_size = self.mock_max_write_batch_sizes.get( host, common.MAX_WRITE_BATCH_SIZE) rtt = self.mock_rtts.get(host, 0) # host is like 'a:1'. if host in self.mock_down_hosts: raise NetworkTimeout('mock timeout') elif host in self.mock_standalones: response = { 'ok': 1, 'ismaster': True, 'minWireVersion': min_wire_version, 'maxWireVersion': max_wire_version, 'maxWriteBatchSize': max_write_batch_size} elif host in self.mock_members: ismaster = (host == self.mock_primary) # Simulate a replica set member. response = { 'ok': 1, 'ismaster': ismaster, 'secondary': not ismaster, 'setName': 'rs', 'hosts': self.mock_ismaster_hosts, 'minWireVersion': min_wire_version, 'maxWireVersion': max_wire_version, 'maxWriteBatchSize': max_write_batch_size} if self.mock_primary: response['primary'] = self.mock_primary elif host in self.mock_mongoses: response = { 'ok': 1, 'ismaster': True, 'minWireVersion': min_wire_version, 'maxWireVersion': max_wire_version, 'msg': 'isdbgrid', 'maxWriteBatchSize': max_write_batch_size} else: # In test_internal_ips(), we try to connect to a host listed # in ismaster['hosts'] but not publicly accessible. raise AutoReconnect('Unknown host: %s' % host) return response, rtt def _process_periodic_tasks(self): # Avoid the background thread causing races, e.g. a surprising # reconnect while we're trying to test a disconnected client. pass pymongo-3.6.1/test/test_mongos_load_balancing.py0000644000076600000240000001452513245621354022336 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test MongoClient's mongos load balancing using a mock.""" import sys import threading sys.path[0:0] = [""] from pymongo.errors import AutoReconnect, InvalidOperation from pymongo.server_selectors import writable_server_selector from pymongo.topology_description import TOPOLOGY_TYPE from test import unittest, client_context, MockClientTest from test.pymongo_mocks import MockClient from test.utils import connected, wait_until @client_context.require_connection def setUpModule(): pass class SimpleOp(threading.Thread): def __init__(self, client): super(SimpleOp, self).__init__() self.client = client self.passed = False def run(self): self.client.db.command('ismaster') self.passed = True # No exception raised. def do_simple_op(client, nthreads): threads = [SimpleOp(client) for _ in range(nthreads)] for t in threads: t.start() for t in threads: t.join() for t in threads: assert t.passed def writable_addresses(topology): return set(server.description.address for server in topology.select_servers(writable_server_selector)) class TestMongosLoadBalancing(MockClientTest): def mock_client(self, **kwargs): mock_client = MockClient( standalones=[], members=[], mongoses=['a:1', 'b:2', 'c:3'], host='a:1,b:2,c:3', connect=False, **kwargs) # Latencies in seconds. mock_client.mock_rtts['a:1'] = 0.020 mock_client.mock_rtts['b:2'] = 0.025 mock_client.mock_rtts['c:3'] = 0.045 return mock_client def test_lazy_connect(self): # While connected() ensures we can trigger connection from the main # thread and wait for the monitors, this test triggers connection from # several threads at once to check for data races. nthreads = 10 client = self.mock_client() self.assertEqual(0, len(client.nodes)) # Trigger initial connection. do_simple_op(client, nthreads) wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') def test_reconnect(self): nthreads = 10 client = connected(self.mock_client()) # connected() ensures we've contacted at least one mongos. Wait for # all of them. wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') # Trigger reconnect. client.close() do_simple_op(client, nthreads) wait_until(lambda: len(client.nodes) == 3, 'reconnect to all mongoses') def test_failover(self): nthreads = 10 client = connected(self.mock_client(localThresholdMS=0.001)) wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') # Our chosen mongos goes down. client.kill_host('a:1') # Trigger failover to higher-latency nodes. AutoReconnect should be # raised at most once in each thread. passed = [] def f(): try: client.db.command('ismaster') except AutoReconnect: # Second attempt succeeds. client.db.command('ismaster') passed.append(True) threads = [threading.Thread(target=f) for _ in range(nthreads)] for t in threads: t.start() for t in threads: t.join() self.assertEqual(nthreads, len(passed)) # Down host removed from list. self.assertEqual(2, len(client.nodes)) def test_local_threshold(self): client = connected(self.mock_client(localThresholdMS=30)) self.assertEqual(30, client.local_threshold_ms) wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') topology = client._topology # All are within a 30-ms latency window, see self.mock_client(). self.assertEqual(set([('a', 1), ('b', 2), ('c', 3)]), writable_addresses(topology)) # No error client.admin.command('ismaster') client = connected(self.mock_client(localThresholdMS=0)) self.assertEqual(0, client.local_threshold_ms) # No error client.db.command('ismaster') # Our chosen mongos goes down. client.kill_host('%s:%s' % next(iter(client.nodes))) try: client.db.command('ismaster') except: pass # We eventually connect to a new mongos. def connect_to_new_mongos(): try: return client.db.command('ismaster') except AutoReconnect: pass wait_until(connect_to_new_mongos, 'connect to a new mongos') def test_load_balancing(self): # Although the server selection JSON tests already prove that # select_servers works for sharded topologies, here we do an end-to-end # test of discovering servers' round trip times and configuring # localThresholdMS. client = connected(self.mock_client()) wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') # Prohibited for topology type Sharded. with self.assertRaises(InvalidOperation): client.address topology = client._topology self.assertEqual(TOPOLOGY_TYPE.Sharded, topology.description.topology_type) # a and b are within the 15-ms latency window, see self.mock_client(). self.assertEqual(set([('a', 1), ('b', 2)]), writable_addresses(topology)) client.mock_rtts['a:1'] = 0.045 # Discover only b is within latency window. wait_until(lambda: set([('b', 2)]) == writable_addresses(topology), 'discover server "a" is too far') if __name__ == "__main__": unittest.main() pymongo-3.6.1/test/test_topology.py0000644000076600000240000006651413245621354017720 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the topology module.""" import sys sys.path[0:0] = [""] import threading from bson.py3compat import imap from pymongo import common from pymongo.read_preferences import ReadPreference, Secondary from pymongo.server_type import SERVER_TYPE from pymongo.topology import Topology from pymongo.topology_description import TOPOLOGY_TYPE from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure) from pymongo.ismaster import IsMaster from pymongo.monitor import Monitor from pymongo.pool import PoolOptions from pymongo.server_description import ServerDescription from pymongo.server_selectors import (any_server_selector, writable_server_selector) from pymongo.settings import TopologySettings from test import client_knobs, unittest from test.utils import wait_until class MockSocketInfo(object): def close(self): pass def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass class MockPool(object): def __init__(self, *args, **kwargs): self.pool_id = 0 self._lock = threading.Lock() self.opts = PoolOptions() def get_socket(self, all_credentials): return MockSocketInfo() def return_socket(self, _): pass def reset(self): with self._lock: self.pool_id += 1 def remove_stale_sockets(self): pass class MockMonitor(object): def __init__(self, server_description, topology, pool, topology_settings): self._server_description = server_description self._topology = topology self.opened = False def open(self): self.opened = True def request_check(self): pass def close(self): self.opened = False class SetNameDiscoverySettings(TopologySettings): def get_topology_type(self): return TOPOLOGY_TYPE.ReplicaSetNoPrimary address = ('a', 27017) def create_mock_topology( seeds=None, replica_set_name=None, monitor_class=MockMonitor): partitioned_seeds = list(imap(common.partition_node, seeds or ['a'])) topology_settings = TopologySettings( partitioned_seeds, replica_set_name=replica_set_name, pool_class=MockPool, monitor_class=monitor_class) t = Topology(topology_settings) t.open() return t def got_ismaster(topology, server_address, ismaster_response): server_description = ServerDescription( server_address, IsMaster(ismaster_response), 0) topology.on_change(server_description) def disconnected(topology, server_address): # Create new description of server type Unknown. topology.on_change(ServerDescription(server_address)) def get_server(topology, hostname): return topology.get_server_by_address((hostname, 27017)) def get_type(topology, hostname): return get_server(topology, hostname).description.server_type def get_monitor(topology, hostname): return get_server(topology, hostname)._monitor class TopologyTest(unittest.TestCase): """Disables periodic monitoring, to make tests deterministic.""" def setUp(self): super(TopologyTest, self).setUp() self.client_knobs = client_knobs(heartbeat_frequency=999999) self.client_knobs.enable() self.addCleanup(self.client_knobs.disable) # Use assertRaisesRegex if available, otherwise use Python 2.7's # deprecated assertRaisesRegexp, with a 'p'. if not hasattr(unittest.TestCase, 'assertRaisesRegex'): TopologyTest.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp class TestTopologyConfiguration(TopologyTest): def test_timeout_configuration(self): pool_options = PoolOptions(connect_timeout=1, socket_timeout=2) topology_settings = TopologySettings(pool_options=pool_options) t = Topology(topology_settings=topology_settings) t.open() # Get the default server. server = t.get_server_by_address(('localhost', 27017)) # The pool for application operations obeys our settings. self.assertEqual(1, server._pool.opts.connect_timeout) self.assertEqual(2, server._pool.opts.socket_timeout) # The pool for monitoring operations uses our connect_timeout as both # its connect_timeout and its socket_timeout. monitor = server._monitor self.assertEqual(1, monitor._pool.opts.connect_timeout) self.assertEqual(1, monitor._pool.opts.socket_timeout) # The monitor, not its pool, is responsible for calling ismaster. self.assertFalse(monitor._pool.handshake) class TestSingleServerTopology(TopologyTest): def test_direct_connection(self): for server_type, ismaster_response in [ (SERVER_TYPE.RSPrimary, { 'ok': 1, 'ismaster': True, 'hosts': ['a'], 'setName': 'rs', 'maxWireVersion': 6}), (SERVER_TYPE.RSSecondary, { 'ok': 1, 'ismaster': False, 'secondary': True, 'hosts': ['a'], 'setName': 'rs', 'maxWireVersion': 6}), (SERVER_TYPE.Mongos, { 'ok': 1, 'ismaster': True, 'msg': 'isdbgrid', 'maxWireVersion': 6}), (SERVER_TYPE.RSArbiter, { 'ok': 1, 'ismaster': False, 'arbiterOnly': True, 'hosts': ['a'], 'setName': 'rs', 'maxWireVersion': 6}), (SERVER_TYPE.Standalone, { 'ok': 1, 'ismaster': True, 'maxWireVersion': 6}), # Slave. (SERVER_TYPE.Standalone, { 'ok': 1, 'ismaster': False, 'maxWireVersion': 6}), ]: t = create_mock_topology() # Can't select a server while the only server is of type Unknown. with self.assertRaisesRegex(ConnectionFailure, 'No servers found yet'): t.select_servers(any_server_selector, server_selection_timeout=0) got_ismaster(t, address, ismaster_response) # Topology type never changes. self.assertEqual(TOPOLOGY_TYPE.Single, t.description.topology_type) # No matter whether the server is writable, # select_servers() returns it. s = t.select_server(writable_server_selector) self.assertEqual(server_type, s.description.server_type) # Topology type single is always readable and writable regardless # of server type or state. self.assertEqual(t.description.topology_type_name, 'Single') self.assertTrue(t.description.has_writable_server()) self.assertTrue(t.description.has_readable_server()) self.assertTrue(t.description.has_readable_server(Secondary())) self.assertTrue(t.description.has_readable_server( Secondary(tag_sets=[{'tag': 'does-not-exist'}]))) def test_reopen(self): t = create_mock_topology() # Additional calls are permitted. t.open() t.open() def test_unavailable_seed(self): t = create_mock_topology() disconnected(t, address) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'a')) def test_round_trip_time(self): round_trip_time = 125 available = True class TestMonitor(Monitor): def _check_with_socket(self, *args, **kwargs): if available: return (IsMaster({'ok': 1, 'maxWireVersion': 6}), round_trip_time) else: raise AutoReconnect('mock monitor error') t = create_mock_topology(monitor_class=TestMonitor) s = t.select_server(writable_server_selector) self.assertEqual(125, s.description.round_trip_time) round_trip_time = 25 t.request_check_all() # Exponential weighted average: .8 * 125 + .2 * 25 = 105. self.assertAlmostEqual(105, s.description.round_trip_time) # The server is temporarily down. available = False t.request_check_all() def raises_err(): try: t.select_server(writable_server_selector, server_selection_timeout=0.1) except ConnectionFailure: return True else: return False wait_until(raises_err, 'discover server is down') self.assertIsNone(s.description.round_trip_time) # Bring it back, RTT is now 20 milliseconds. available = True round_trip_time = 20 def new_average(): # We reset the average to the most recent measurement. description = s.description return (description.round_trip_time is not None and round(abs(20 - description.round_trip_time), 7) == 0) tries = 0 while not new_average(): t.request_check_all() tries += 1 if tries > 10: self.fail("Didn't ever calculate correct new average") class TestMultiServerTopology(TopologyTest): def test_readable_writable(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b']}) self.assertTrue( t.description.topology_type_name, 'ReplicaSetWithPrimary') self.assertTrue(t.description.has_writable_server()) self.assertTrue(t.description.has_readable_server()) self.assertTrue( t.description.has_readable_server(Secondary())) self.assertFalse( t.description.has_readable_server( Secondary(tag_sets=[{'tag': 'exists'}]))) t = create_mock_topology(replica_set_name='rs') got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': False, 'secondary': False, 'setName': 'rs', 'hosts': ['a', 'b']}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b']}) self.assertTrue( t.description.topology_type_name, 'ReplicaSetNoPrimary') self.assertFalse(t.description.has_writable_server()) self.assertFalse(t.description.has_readable_server()) self.assertTrue( t.description.has_readable_server(Secondary())) self.assertFalse( t.description.has_readable_server( Secondary(tag_sets=[{'tag': 'exists'}]))) t = create_mock_topology(replica_set_name='rs') got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b'], 'tags': {'tag': 'exists'}}) self.assertTrue( t.description.topology_type_name, 'ReplicaSetWithPrimary') self.assertTrue(t.description.has_writable_server()) self.assertTrue(t.description.has_readable_server()) self.assertTrue( t.description.has_readable_server(Secondary())) self.assertTrue( t.description.has_readable_server( Secondary(tag_sets=[{'tag': 'exists'}]))) def test_close(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b']}) self.assertEqual(SERVER_TYPE.RSPrimary, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.RSSecondary, get_type(t, 'b')) self.assertTrue(get_monitor(t, 'a').opened) self.assertTrue(get_monitor(t, 'b').opened) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetWithPrimary, t.description.topology_type) t.close() self.assertEqual(2, len(t.description.server_descriptions())) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'b')) self.assertFalse(get_monitor(t, 'a').opened) self.assertFalse(get_monitor(t, 'b').opened) self.assertEqual('rs', t.description.replica_set_name) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetNoPrimary, t.description.topology_type) # A closed topology should not be updated when receiving an isMaster. got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b', 'c']}) self.assertEqual(2, len(t.description.server_descriptions())) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'b')) self.assertFalse(get_monitor(t, 'a').opened) self.assertFalse(get_monitor(t, 'b').opened) # Server c should not have been added. self.assertEqual(None, get_server(t, 'c')) self.assertEqual('rs', t.description.replica_set_name) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetNoPrimary, t.description.topology_type) def test_reset_server(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b']}) t.reset_server(('a', 27017)) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.RSSecondary, get_type(t, 'b')) self.assertEqual('rs', t.description.replica_set_name) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetNoPrimary, t.description.topology_type) got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) self.assertEqual(SERVER_TYPE.RSPrimary, get_type(t, 'a')) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetWithPrimary, t.description.topology_type) t.reset_server(('b', 27017)) self.assertEqual(SERVER_TYPE.RSPrimary, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'b')) self.assertEqual('rs', t.description.replica_set_name) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetWithPrimary, t.description.topology_type) def test_reset_removed_server(self): t = create_mock_topology(replica_set_name='rs') # No error resetting a server not in the TopologyDescription. t.reset_server(('b', 27017)) # Server was *not* added as type Unknown. self.assertFalse(t.has_server(('b', 27017))) def test_discover_set_name_from_primary(self): # Discovering a replica set without the setName supplied by the user # is not yet supported by MongoClient, but Topology can do it. topology_settings = SetNameDiscoverySettings( seeds=[address], pool_class=MockPool, monitor_class=MockMonitor) t = Topology(topology_settings) self.assertEqual(t.description.replica_set_name, None) self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetNoPrimary) t.open() got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a']}) self.assertEqual(t.description.replica_set_name, 'rs') self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetWithPrimary) # Another response from the primary. Tests the code that processes # primary response when topology type is already ReplicaSetWithPrimary. got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a']}) # No change. self.assertEqual(t.description.replica_set_name, 'rs') self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetWithPrimary) def test_discover_set_name_from_secondary(self): # Discovering a replica set without the setName supplied by the user # is not yet supported by MongoClient, but Topology can do it. topology_settings = SetNameDiscoverySettings( seeds=[address], pool_class=MockPool, monitor_class=MockMonitor) t = Topology(topology_settings) self.assertEqual(t.description.replica_set_name, None) self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetNoPrimary) t.open() got_ismaster(t, address, { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a']}) self.assertEqual(t.description.replica_set_name, 'rs') self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetNoPrimary) def test_wire_version(self): t = create_mock_topology(replica_set_name='rs') t.description.check_compatible() # No error. got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a']}) # Use defaults. server = t.get_server_by_address(address) self.assertEqual(server.description.min_wire_version, 0) self.assertEqual(server.description.max_wire_version, 0) got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a'], 'minWireVersion': 1, 'maxWireVersion': 5}) self.assertEqual(server.description.min_wire_version, 1) self.assertEqual(server.description.max_wire_version, 5) # Incompatible. got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a'], 'minWireVersion': 11, 'maxWireVersion': 12}) try: t.select_servers(any_server_selector) except ConfigurationError as e: # Error message should say which server failed and why. self.assertEqual( str(e), "Server at a:27017 requires wire version 11, but this version " "of PyMongo only supports up to %d." % (common.MAX_SUPPORTED_WIRE_VERSION,)) else: self.fail('No error with incompatible wire version') # Incompatible. got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a'], 'minWireVersion': 0, 'maxWireVersion': 0}) try: t.select_servers(any_server_selector) except ConfigurationError as e: # Error message should say which server failed and why. self.assertEqual( str(e), "Server at a:27017 reports wire version 0, but this version " "of PyMongo requires at least %d (MongoDB %s)." % (common.MIN_SUPPORTED_WIRE_VERSION, common.MIN_SUPPORTED_SERVER_VERSION)) else: self.fail('No error with incompatible wire version') def test_max_write_batch_size(self): t = create_mock_topology(seeds=['a', 'b'], replica_set_name='rs') def write_batch_size(): s = t.select_server(writable_server_selector) return s.description.max_write_batch_size got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b'], 'maxWireVersion': 6, 'maxWriteBatchSize': 1}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b'], 'maxWireVersion': 6, 'maxWriteBatchSize': 2}) # Uses primary's max batch size. self.assertEqual(1, write_batch_size()) # b becomes primary. got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b'], 'maxWireVersion': 6, 'maxWriteBatchSize': 2}) self.assertEqual(2, write_batch_size()) def wait_for_master(topology): """Wait for a Topology to discover a writable server. If the monitor is currently calling ismaster, a blocking call to select_server from this thread can trigger a spurious wake of the monitor thread. In applications this is harmless but it would break some tests, so we pass server_selection_timeout=0 and poll instead. """ def get_master(): try: return topology.select_server(writable_server_selector, 0) except ConnectionFailure: return None return wait_until(get_master, 'find master') class TestTopologyErrors(TopologyTest): # Errors when calling ismaster. def test_pool_reset(self): # ismaster succeeds at first, then always raises socket error. ismaster_count = [0] class TestMonitor(Monitor): def _check_with_socket(self, *args, **kwargs): ismaster_count[0] += 1 if ismaster_count[0] == 1: return IsMaster({'ok': 1, 'maxWireVersion': 6}), 0 else: raise AutoReconnect('mock monitor error') t = create_mock_topology(monitor_class=TestMonitor) server = wait_for_master(t) self.assertEqual(1, ismaster_count[0]) pool_id = server.pool.pool_id # Pool is reset by ismaster failure. t.request_check_all() self.assertNotEqual(pool_id, server.pool.pool_id) def test_ismaster_retry(self): # ismaster succeeds at first, then raises socket error, then succeeds. ismaster_count = [0] class TestMonitor(Monitor): def _check_with_socket(self, *args, **kwargs): ismaster_count[0] += 1 if ismaster_count[0] in (1, 3): return IsMaster({'ok': 1, 'maxWireVersion': 6}), 0 else: raise AutoReconnect('mock monitor error') t = create_mock_topology(monitor_class=TestMonitor) server = wait_for_master(t) self.assertEqual(1, ismaster_count[0]) self.assertEqual(SERVER_TYPE.Standalone, server.description.server_type) # Second ismaster call, then immediately the third. t.request_check_all() self.assertEqual(3, ismaster_count[0]) self.assertEqual(SERVER_TYPE.Standalone, get_type(t, 'a')) def test_internal_monitor_error(self): exception = AssertionError('internal error') class TestMonitor(Monitor): def _check_with_socket(self, *args, **kwargs): raise exception t = create_mock_topology(monitor_class=TestMonitor) with self.assertRaisesRegex(ConnectionFailure, 'internal error'): t.select_server(any_server_selector, server_selection_timeout=0.5) class TestServerSelectionErrors(TopologyTest): def assertMessage(self, message, topology, selector=any_server_selector): with self.assertRaises(ConnectionFailure) as context: topology.select_server(selector, server_selection_timeout=0) self.assertEqual(message, str(context.exception)) def test_no_primary(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, address, { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a']}) self.assertMessage('No replica set members match selector "Primary()"', t, ReadPreference.PRIMARY) self.assertMessage('No primary available for writes', t, writable_server_selector) def test_no_secondary(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a']}) self.assertMessage( 'No replica set members match selector' ' "Secondary(tag_sets=None, max_staleness=-1)"', t, ReadPreference.SECONDARY) self.assertMessage( "No replica set members match selector" " \"Secondary(tag_sets=[{'dc': 'ny'}], max_staleness=-1)\"", t, Secondary(tag_sets=[{'dc': 'ny'}])) def test_bad_replica_set_name(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, address, { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'wrong', 'hosts': ['a']}) self.assertMessage( 'No replica set members available for replica set name "rs"', t) def test_multiple_standalones(self): # Standalones are removed from a topology with multiple seeds. t = create_mock_topology(seeds=['a', 'b']) got_ismaster(t, ('a', 27017), {'ok': 1}) got_ismaster(t, ('b', 27017), {'ok': 1}) self.assertMessage('No servers available', t) def test_no_mongoses(self): # Standalones are removed from a topology with multiple seeds. t = create_mock_topology(seeds=['a', 'b']) # Discover a mongos and change topology type to Sharded. got_ismaster(t, ('a', 27017), {'ok': 1, 'msg': 'isdbgrid'}) # Oops, both servers are standalone now. Remove them. got_ismaster(t, ('a', 27017), {'ok': 1}) got_ismaster(t, ('b', 27017), {'ok': 1}) self.assertMessage('No mongoses available', t) if __name__ == "__main__": unittest.main() pymongo-3.6.1/MANIFEST.in0000644000076600000240000000060413245617773015210 0ustar shanestaff00000000000000include README.rst include LICENSE include THIRD-PARTY-NOTICES include ez_setup.py recursive-include doc *.rst recursive-include doc *.py recursive-include doc *.conf recursive-include doc *.css recursive-include doc *.js recursive-include doc *.png recursive-include tools *.py include tools/README.rst recursive-include test *.pem recursive-include test *.py recursive-include bson *.h pymongo-3.6.1/pymongo/0000755000076600000240000000000013246104133015120 5ustar shanestaff00000000000000pymongo-3.6.1/pymongo/topology_description.py0000644000076600000240000004756713245621354022004 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Represent a deployment of MongoDB servers.""" from collections import namedtuple from pymongo import common from pymongo.errors import ConfigurationError from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.server_selectors import Selection from pymongo.server_type import SERVER_TYPE TOPOLOGY_TYPE = namedtuple('TopologyType', ['Single', 'ReplicaSetNoPrimary', 'ReplicaSetWithPrimary', 'Sharded', 'Unknown'])(*range(5)) class TopologyDescription(object): def __init__(self, topology_type, server_descriptions, replica_set_name, max_set_version, max_election_id, topology_settings): """Representation of a deployment of MongoDB servers. :Parameters: - `topology_type`: initial type - `server_descriptions`: dict of (address, ServerDescription) for all seeds - `replica_set_name`: replica set name or None - `max_set_version`: greatest setVersion seen from a primary, or None - `max_election_id`: greatest electionId seen from a primary, or None - `topology_settings`: a TopologySettings """ self._topology_type = topology_type self._replica_set_name = replica_set_name self._server_descriptions = server_descriptions self._max_set_version = max_set_version self._max_election_id = max_election_id # The heartbeat_frequency is used in staleness estimates. self._topology_settings = topology_settings # Is PyMongo compatible with all servers' wire protocols? self._incompatible_err = None for s in self._server_descriptions.values(): if not s.is_server_type_known: continue # s.min/max_wire_version is the server's wire protocol. # MIN/MAX_SUPPORTED_WIRE_VERSION is what PyMongo supports. server_too_new = ( # Server too new. s.min_wire_version is not None and s.min_wire_version > common.MAX_SUPPORTED_WIRE_VERSION) server_too_old = ( # Server too old. s.max_wire_version is not None and s.max_wire_version < common.MIN_SUPPORTED_WIRE_VERSION) if server_too_new: self._incompatible_err = ( "Server at %s:%d requires wire version %d, but this " "version of PyMongo only supports up to %d." % (s.address[0], s.address[1], s.min_wire_version, common.MAX_SUPPORTED_WIRE_VERSION)) elif server_too_old: self._incompatible_err = ( "Server at %s:%d reports wire version %d, but this " "version of PyMongo requires at least %d (MongoDB %s)." % (s.address[0], s.address[1], s.max_wire_version, common.MIN_SUPPORTED_WIRE_VERSION, common.MIN_SUPPORTED_SERVER_VERSION)) break # Server Discovery And Monitoring Spec: Whenever a client updates the # TopologyDescription from an ismaster response, it MUST set # TopologyDescription.logicalSessionTimeoutMinutes to the smallest # logicalSessionTimeoutMinutes value among ServerDescriptions of all # data-bearing server types. If any have a null # logicalSessionTimeoutMinutes, then # TopologyDescription.logicalSessionTimeoutMinutes MUST be set to null. readable_servers = self.readable_servers if not readable_servers: self._ls_timeout_minutes = None elif any(s.logical_session_timeout_minutes is None for s in readable_servers): self._ls_timeout_minutes = None else: self._ls_timeout_minutes = min(s.logical_session_timeout_minutes for s in readable_servers) def check_compatible(self): """Raise ConfigurationError if any server is incompatible. A server is incompatible if its wire protocol version range does not overlap with PyMongo's. """ if self._incompatible_err: raise ConfigurationError(self._incompatible_err) def has_server(self, address): return address in self._server_descriptions def reset_server(self, address): """A copy of this description, with one server marked Unknown.""" return updated_topology_description(self, ServerDescription(address)) def reset(self): """A copy of this description, with all servers marked Unknown.""" if self._topology_type == TOPOLOGY_TYPE.ReplicaSetWithPrimary: topology_type = TOPOLOGY_TYPE.ReplicaSetNoPrimary else: topology_type = self._topology_type # The default ServerDescription's type is Unknown. sds = dict((address, ServerDescription(address)) for address in self._server_descriptions) return TopologyDescription( topology_type, sds, self._replica_set_name, self._max_set_version, self._max_election_id, self._topology_settings) def server_descriptions(self): """Dict of (address, :class:`~pymongo.server_description.ServerDescription`).""" return self._server_descriptions.copy() @property def topology_type(self): """The type of this topology.""" return self._topology_type @property def topology_type_name(self): """The topology type as a human readable string. .. versionadded:: 3.4 """ return TOPOLOGY_TYPE._fields[self._topology_type] @property def replica_set_name(self): """The replica set name.""" return self._replica_set_name @property def max_set_version(self): """Greatest setVersion seen from a primary, or None.""" return self._max_set_version @property def max_election_id(self): """Greatest electionId seen from a primary, or None.""" return self._max_election_id @property def logical_session_timeout_minutes(self): """Minimum logical session timeout, or None.""" return self._ls_timeout_minutes @property def known_servers(self): """List of Servers of types besides Unknown.""" return [s for s in self._server_descriptions.values() if s.is_server_type_known] @property def has_known_servers(self): """Whether there are any Servers of types besides Unknown.""" return any(s for s in self._server_descriptions.values() if s.is_server_type_known) @property def readable_servers(self): """List of readable Servers.""" return [s for s in self._server_descriptions.values() if s.is_readable] @property def common_wire_version(self): """Minimum of all servers' max wire versions, or None.""" servers = self.known_servers if servers: return min(s.max_wire_version for s in self.known_servers) return None @property def heartbeat_frequency(self): return self._topology_settings.heartbeat_frequency def apply_selector(self, selector, address): def apply_local_threshold(selection): if not selection: return [] settings = self._topology_settings # Round trip time in seconds. fastest = min( s.round_trip_time for s in selection.server_descriptions) threshold = settings.local_threshold_ms / 1000.0 return [s for s in selection.server_descriptions if (s.round_trip_time - fastest) <= threshold] if getattr(selector, 'min_wire_version', 0): common_wv = self.common_wire_version if common_wv and common_wv < selector.min_wire_version: raise ConfigurationError( "%s requires min wire version %d, but topology's min" " wire version is %d" % (selector, selector.min_wire_version, common_wv)) if self.topology_type == TOPOLOGY_TYPE.Single: # Ignore the selector. return self.known_servers elif address: description = self.server_descriptions().get(address) return [description] if description else [] elif self.topology_type == TOPOLOGY_TYPE.Sharded: # Ignore the read preference, but apply localThresholdMS. return apply_local_threshold( Selection.from_topology_description(self)) else: return apply_local_threshold( selector(Selection.from_topology_description(self))) def has_readable_server(self, read_preference=ReadPreference.PRIMARY): """Does this topology have any readable servers available matching the given read preference? :Parameters: - `read_preference`: an instance of a read preference from :mod:`~pymongo.read_preferences`. Defaults to :attr:`~pymongo.read_preferences.ReadPreference.PRIMARY`. .. note:: When connected directly to a single server this method always returns ``True``. .. versionadded:: 3.4 """ common.validate_read_preference("read_preference", read_preference) return any(self.apply_selector(read_preference, None)) def has_writable_server(self): """Does this topology have a writable server available? .. note:: When connected directly to a single server this method always returns ``True``. .. versionadded:: 3.4 """ return self.has_readable_server(ReadPreference.PRIMARY) # If topology type is Unknown and we receive an ismaster response, what should # the new topology type be? _SERVER_TYPE_TO_TOPOLOGY_TYPE = { SERVER_TYPE.Mongos: TOPOLOGY_TYPE.Sharded, SERVER_TYPE.RSPrimary: TOPOLOGY_TYPE.ReplicaSetWithPrimary, SERVER_TYPE.RSSecondary: TOPOLOGY_TYPE.ReplicaSetNoPrimary, SERVER_TYPE.RSArbiter: TOPOLOGY_TYPE.ReplicaSetNoPrimary, SERVER_TYPE.RSOther: TOPOLOGY_TYPE.ReplicaSetNoPrimary, } def updated_topology_description(topology_description, server_description): """Return an updated copy of a TopologyDescription. :Parameters: - `topology_description`: the current TopologyDescription - `server_description`: a new ServerDescription that resulted from an ismaster call Called after attempting (successfully or not) to call ismaster on the server at server_description.address. Does not modify topology_description. """ address = server_description.address # These values will be updated, if necessary, to form the new # TopologyDescription. topology_type = topology_description.topology_type set_name = topology_description.replica_set_name max_set_version = topology_description.max_set_version max_election_id = topology_description.max_election_id server_type = server_description.server_type # Don't mutate the original dict of server descriptions; copy it. sds = topology_description.server_descriptions() # Replace this server's description with the new one. sds[address] = server_description if topology_type == TOPOLOGY_TYPE.Single: # Single type never changes. return TopologyDescription( TOPOLOGY_TYPE.Single, sds, set_name, max_set_version, max_election_id, topology_description._topology_settings) if topology_type == TOPOLOGY_TYPE.Unknown: if server_type == SERVER_TYPE.Standalone: sds.pop(address) elif server_type not in (SERVER_TYPE.Unknown, SERVER_TYPE.RSGhost): topology_type = _SERVER_TYPE_TO_TOPOLOGY_TYPE[server_type] if topology_type == TOPOLOGY_TYPE.Sharded: if server_type not in (SERVER_TYPE.Mongos, SERVER_TYPE.Unknown): sds.pop(address) elif topology_type == TOPOLOGY_TYPE.ReplicaSetNoPrimary: if server_type in (SERVER_TYPE.Standalone, SERVER_TYPE.Mongos): sds.pop(address) elif server_type == SERVER_TYPE.RSPrimary: (topology_type, set_name, max_set_version, max_election_id) = _update_rs_from_primary(sds, set_name, server_description, max_set_version, max_election_id) elif server_type in ( SERVER_TYPE.RSSecondary, SERVER_TYPE.RSArbiter, SERVER_TYPE.RSOther): topology_type, set_name = _update_rs_no_primary_from_member( sds, set_name, server_description) elif topology_type == TOPOLOGY_TYPE.ReplicaSetWithPrimary: if server_type in (SERVER_TYPE.Standalone, SERVER_TYPE.Mongos): sds.pop(address) topology_type = _check_has_primary(sds) elif server_type == SERVER_TYPE.RSPrimary: (topology_type, set_name, max_set_version, max_election_id) = _update_rs_from_primary(sds, set_name, server_description, max_set_version, max_election_id) elif server_type in ( SERVER_TYPE.RSSecondary, SERVER_TYPE.RSArbiter, SERVER_TYPE.RSOther): topology_type = _update_rs_with_primary_from_member( sds, set_name, server_description) else: # Server type is Unknown or RSGhost: did we just lose the primary? topology_type = _check_has_primary(sds) # Return updated copy. return TopologyDescription(topology_type, sds, set_name, max_set_version, max_election_id, topology_description._topology_settings) def _update_rs_from_primary( sds, replica_set_name, server_description, max_set_version, max_election_id): """Update topology description from a primary's ismaster response. Pass in a dict of ServerDescriptions, current replica set name, the ServerDescription we are processing, and the TopologyDescription's max_set_version and max_election_id if any. Returns (new topology type, new replica_set_name, new max_set_version, new max_election_id). """ if replica_set_name is None: replica_set_name = server_description.replica_set_name elif replica_set_name != server_description.replica_set_name: # We found a primary but it doesn't have the replica_set_name # provided by the user. sds.pop(server_description.address) return (_check_has_primary(sds), replica_set_name, max_set_version, max_election_id) max_election_tuple = max_set_version, max_election_id if None not in server_description.election_tuple: if (None not in max_election_tuple and max_election_tuple > server_description.election_tuple): # Stale primary, set to type Unknown. address = server_description.address sds[address] = ServerDescription(address) return (_check_has_primary(sds), replica_set_name, max_set_version, max_election_id) max_election_id = server_description.election_id if (server_description.set_version is not None and (max_set_version is None or server_description.set_version > max_set_version)): max_set_version = server_description.set_version # We've heard from the primary. Is it the same primary as before? for server in sds.values(): if (server.server_type is SERVER_TYPE.RSPrimary and server.address != server_description.address): # Reset old primary's type to Unknown. sds[server.address] = ServerDescription(server.address) # There can be only one prior primary. break # Discover new hosts from this primary's response. for new_address in server_description.all_hosts: if new_address not in sds: sds[new_address] = ServerDescription(new_address) # Remove hosts not in the response. for addr in set(sds) - server_description.all_hosts: sds.pop(addr) # If the host list differs from the seed list, we may not have a primary # after all. return (_check_has_primary(sds), replica_set_name, max_set_version, max_election_id) def _update_rs_with_primary_from_member( sds, replica_set_name, server_description): """RS with known primary. Process a response from a non-primary. Pass in a dict of ServerDescriptions, current replica set name, and the ServerDescription we are processing. Returns new topology type. """ assert replica_set_name is not None if replica_set_name != server_description.replica_set_name: sds.pop(server_description.address) elif (server_description.me and server_description.address != server_description.me): sds.pop(server_description.address) # Had this member been the primary? return _check_has_primary(sds) def _update_rs_no_primary_from_member( sds, replica_set_name, server_description): """RS without known primary. Update from a non-primary's response. Pass in a dict of ServerDescriptions, current replica set name, and the ServerDescription we are processing. Returns (new topology type, new replica_set_name). """ topology_type = TOPOLOGY_TYPE.ReplicaSetNoPrimary if replica_set_name is None: replica_set_name = server_description.replica_set_name elif replica_set_name != server_description.replica_set_name: sds.pop(server_description.address) return topology_type, replica_set_name # This isn't the primary's response, so don't remove any servers # it doesn't report. Only add new servers. for address in server_description.all_hosts: if address not in sds: sds[address] = ServerDescription(address) if (server_description.me and server_description.address != server_description.me): sds.pop(server_description.address) return topology_type, replica_set_name def _check_has_primary(sds): """Current topology type is ReplicaSetWithPrimary. Is primary still known? Pass in a dict of ServerDescriptions. Returns new topology type. """ for s in sds.values(): if s.server_type == SERVER_TYPE.RSPrimary: return TOPOLOGY_TYPE.ReplicaSetWithPrimary else: return TOPOLOGY_TYPE.ReplicaSetNoPrimary pymongo-3.6.1/pymongo/ssl_support.py0000644000076600000240000001602013245621354020076 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Support for SSL in PyMongo.""" import atexit import sys import threading HAVE_SSL = True try: import ssl except ImportError: HAVE_SSL = False HAVE_CERTIFI = False try: import certifi HAVE_CERTIFI = True except ImportError: pass HAVE_WINCERTSTORE = False try: from wincertstore import CertFile HAVE_WINCERTSTORE = True except ImportError: pass from bson.py3compat import string_type from pymongo.errors import ConfigurationError _WINCERTSLOCK = threading.Lock() _WINCERTS = None if HAVE_SSL: try: # Python 2.7.9+, 3.2+, PyPy 2.5.1+, etc. from ssl import SSLContext except ImportError: from pymongo.ssl_context import SSLContext def validate_cert_reqs(option, value): """Validate the cert reqs are valid. It must be None or one of the three values ``ssl.CERT_NONE``, ``ssl.CERT_OPTIONAL`` or ``ssl.CERT_REQUIRED``. """ if value is None: return value elif isinstance(value, string_type) and hasattr(ssl, value): value = getattr(ssl, value) if value in (ssl.CERT_NONE, ssl.CERT_OPTIONAL, ssl.CERT_REQUIRED): return value raise ValueError("The value of %s must be one of: " "`ssl.CERT_NONE`, `ssl.CERT_OPTIONAL` or " "`ssl.CERT_REQUIRED`" % (option,)) def _load_wincerts(): """Set _WINCERTS to an instance of wincertstore.Certfile.""" global _WINCERTS certfile = CertFile() certfile.addstore("CA") certfile.addstore("ROOT") atexit.register(certfile.close) _WINCERTS = certfile # XXX: Possible future work. # - OCSP? Not supported by python at all. # http://bugs.python.org/issue17123 # - Adding an ssl_context keyword argument to MongoClient? This might # be useful for sites that have unusual requirements rather than # trying to expose every SSLContext option through a keyword/uri # parameter. def get_ssl_context(*args): """Create and return an SSLContext object.""" certfile, keyfile, passphrase, ca_certs, cert_reqs, crlfile = args # Note PROTOCOL_SSLv23 is about the most misleading name imaginable. # This configures the server and client to negotiate the # highest protocol version they both support. A very good thing. # PROTOCOL_TLS_CLIENT was added in CPython 3.6, deprecating # PROTOCOL_SSLv23. ctx = SSLContext( getattr(ssl, "PROTOCOL_TLS_CLIENT", ssl.PROTOCOL_SSLv23)) # SSLContext.check_hostname was added in CPython 2.7.9 and 3.4. # PROTOCOL_TLS_CLIENT enables it by default. Using it # requires passing server_hostname to wrap_socket, which we already # do for SNI support. To support older versions of Python we have to # call match_hostname directly, so we disable check_hostname explicitly # to avoid calling match_hostname twice. if hasattr(ctx, "check_hostname"): ctx.check_hostname = False if hasattr(ctx, "options"): # Explicitly disable SSLv2, SSLv3 and TLS compression. Note that # up to date versions of MongoDB 2.4 and above already disable # SSLv2 and SSLv3, python disables SSLv2 by default in >= 2.7.7 # and >= 3.3.4 and SSLv3 in >= 3.4.3. There is no way for us to do # any of this explicitly for python 2.6 or 2.7 before 2.7.9. ctx.options |= getattr(ssl, "OP_NO_SSLv2", 0) ctx.options |= getattr(ssl, "OP_NO_SSLv3", 0) # OpenSSL >= 1.0.0 ctx.options |= getattr(ssl, "OP_NO_COMPRESSION", 0) if certfile is not None: try: if passphrase is not None: vi = sys.version_info # Since python just added a new parameter to an existing method # this seems to be about the best we can do. if (vi[0] == 2 and vi < (2, 7, 9) or vi[0] == 3 and vi < (3, 3)): raise ConfigurationError( "Support for ssl_pem_passphrase requires " "python 2.7.9+ (pypy 2.5.1+) or 3.3+") ctx.load_cert_chain(certfile, keyfile, passphrase) else: ctx.load_cert_chain(certfile, keyfile) except ssl.SSLError as exc: raise ConfigurationError( "Private key doesn't match certificate: %s" % (exc,)) if crlfile is not None: if not hasattr(ctx, "verify_flags"): raise ConfigurationError( "Support for ssl_crlfile requires " "python 2.7.9+ (pypy 2.5.1+) or 3.4+") # Match the server's behavior. ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF ctx.load_verify_locations(crlfile) if ca_certs is not None: ctx.load_verify_locations(ca_certs) elif cert_reqs != ssl.CERT_NONE: # CPython >= 2.7.9 or >= 3.4.0, pypy >= 2.5.1 if hasattr(ctx, "load_default_certs"): ctx.load_default_certs() # Python >= 3.2.0, useless on Windows. elif (sys.platform != "win32" and hasattr(ctx, "set_default_verify_paths")): ctx.set_default_verify_paths() elif sys.platform == "win32" and HAVE_WINCERTSTORE: with _WINCERTSLOCK: if _WINCERTS is None: _load_wincerts() ctx.load_verify_locations(_WINCERTS.name) elif HAVE_CERTIFI: ctx.load_verify_locations(certifi.where()) else: raise ConfigurationError( "`ssl_cert_reqs` is not ssl.CERT_NONE and no system " "CA certificates could be loaded. `ssl_ca_certs` is " "required.") ctx.verify_mode = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs return ctx else: def validate_cert_reqs(option, dummy): """No ssl module, raise ConfigurationError.""" raise ConfigurationError("The value of %s is set but can't be " "validated. The ssl module is not available" % (option,)) def get_ssl_context(*dummy): """No ssl module, raise ConfigurationError.""" raise ConfigurationError("The ssl module is not available.") pymongo-3.6.1/pymongo/auth.py0000644000076600000240000004260313245621354016450 0ustar shanestaff00000000000000# Copyright 2013-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Authentication helpers.""" import hmac import socket try: from urllib import quote except ImportError: from urllib.parse import quote HAVE_KERBEROS = True _USE_PRINCIPAL = False try: import winkerberos as kerberos if tuple(map(int, kerberos.__version__.split('.')[:2])) >= (0, 5): _USE_PRINCIPAL = True except ImportError: try: import kerberos except ImportError: HAVE_KERBEROS = False from base64 import standard_b64decode, standard_b64encode from collections import namedtuple from hashlib import md5, sha1 from random import SystemRandom from bson.binary import Binary from bson.py3compat import b, string_type, _unicode, PY3 from bson.son import SON from pymongo.errors import ConfigurationError, OperationFailure MECHANISMS = frozenset( ['GSSAPI', 'MONGODB-CR', 'MONGODB-X509', 'PLAIN', 'SCRAM-SHA-1', 'DEFAULT']) """The authentication mechanisms supported by PyMongo.""" MongoCredential = namedtuple( 'MongoCredential', ['mechanism', 'source', 'username', 'password', 'mechanism_properties']) """A hashable namedtuple of values used for authentication.""" GSSAPIProperties = namedtuple('GSSAPIProperties', ['service_name', 'canonicalize_host_name', 'service_realm']) """Mechanism properties for GSSAPI authentication.""" def _build_credentials_tuple(mech, source, user, passwd, extra): """Build and return a mechanism specific credentials tuple. """ user = _unicode(user) if user is not None else None password = passwd if passwd is None else _unicode(passwd) if mech == 'GSSAPI': properties = extra.get('authmechanismproperties', {}) service_name = properties.get('SERVICE_NAME', 'mongodb') canonicalize = properties.get('CANONICALIZE_HOST_NAME', False) service_realm = properties.get('SERVICE_REALM') props = GSSAPIProperties(service_name=service_name, canonicalize_host_name=canonicalize, service_realm=service_realm) # Source is always $external. return MongoCredential(mech, '$external', user, password, props) elif mech == 'MONGODB-X509': # user can be None. return MongoCredential(mech, '$external', user, None, None) else: if passwd is None: raise ConfigurationError("A password is required.") return MongoCredential(mech, source, user, password, None) if PY3: def _xor(fir, sec): """XOR two byte strings together (python 3.x).""" return b"".join([bytes([x ^ y]) for x, y in zip(fir, sec)]) _from_bytes = int.from_bytes _to_bytes = int.to_bytes else: from binascii import (hexlify as _hexlify, unhexlify as _unhexlify) def _xor(fir, sec): """XOR two byte strings together (python 2.x).""" return b"".join([chr(ord(x) ^ ord(y)) for x, y in zip(fir, sec)]) def _from_bytes(value, dummy, int=int, _hexlify=_hexlify): """An implementation of int.from_bytes for python 2.x.""" return int(_hexlify(value), 16) def _to_bytes(value, dummy0, dummy1, _unhexlify=_unhexlify): """An implementation of int.to_bytes for python 2.x.""" return _unhexlify('%040x' % value) try: # The fastest option, if it's been compiled to use OpenSSL's HMAC. from backports.pbkdf2 import pbkdf2_hmac def _hi(data, salt, iterations): return pbkdf2_hmac('sha1', data, salt, iterations) except ImportError: try: # Python 2.7.8+, or Python 3.4+. from hashlib import pbkdf2_hmac def _hi(data, salt, iterations): return pbkdf2_hmac('sha1', data, salt, iterations) except ImportError: def _hi(data, salt, iterations): """A simple implementation of PBKDF2.""" mac = hmac.HMAC(data, None, sha1) def _digest(msg, mac=mac): """Get a digest for msg.""" _mac = mac.copy() _mac.update(msg) return _mac.digest() from_bytes = _from_bytes to_bytes = _to_bytes _u1 = _digest(salt + b'\x00\x00\x00\x01') _ui = from_bytes(_u1, 'big') for _ in range(iterations - 1): _u1 = _digest(_u1) _ui ^= from_bytes(_u1, 'big') return to_bytes(_ui, 20, 'big') try: from hmac import compare_digest except ImportError: if PY3: def _xor_bytes(a, b): return a ^ b else: def _xor_bytes(a, b, _ord=ord): return _ord(a) ^ _ord(b) # Python 2.x < 2.7.7 and Python 3.x < 3.3 # References: # - http://bugs.python.org/issue14532 # - http://bugs.python.org/issue14955 # - http://bugs.python.org/issue15061 def compare_digest(a, b, _xor_bytes=_xor_bytes): left = None right = b if len(a) == len(b): left = a result = 0 if len(a) != len(b): left = b result = 1 for x, y in zip(left, right): result |= _xor_bytes(x, y) return result == 0 def _parse_scram_response(response): """Split a scram response into key, value pairs.""" return dict(item.split(b"=", 1) for item in response.split(b",")) def _authenticate_scram_sha1(credentials, sock_info): """Authenticate using SCRAM-SHA-1.""" username = credentials.username password = credentials.password source = credentials.source # Make local _hmac = hmac.HMAC _sha1 = sha1 user = username.encode("utf-8").replace(b"=", b"=3D").replace(b",", b"=2C") nonce = standard_b64encode( (("%s" % (SystemRandom().random(),))[2:]).encode("utf-8")) first_bare = b"n=" + user + b",r=" + nonce cmd = SON([('saslStart', 1), ('mechanism', 'SCRAM-SHA-1'), ('payload', Binary(b"n,," + first_bare)), ('autoAuthorize', 1)]) res = sock_info.command(source, cmd) server_first = res['payload'] parsed = _parse_scram_response(server_first) iterations = int(parsed[b'i']) salt = parsed[b's'] rnonce = parsed[b'r'] if not rnonce.startswith(nonce): raise OperationFailure("Server returned an invalid nonce.") without_proof = b"c=biws,r=" + rnonce salted_pass = _hi(_password_digest(username, password).encode("utf-8"), standard_b64decode(salt), iterations) client_key = _hmac(salted_pass, b"Client Key", _sha1).digest() stored_key = _sha1(client_key).digest() auth_msg = b",".join((first_bare, server_first, without_proof)) client_sig = _hmac(stored_key, auth_msg, _sha1).digest() client_proof = b"p=" + standard_b64encode(_xor(client_key, client_sig)) client_final = b",".join((without_proof, client_proof)) server_key = _hmac(salted_pass, b"Server Key", _sha1).digest() server_sig = standard_b64encode( _hmac(server_key, auth_msg, _sha1).digest()) cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']), ('payload', Binary(client_final))]) res = sock_info.command(source, cmd) parsed = _parse_scram_response(res['payload']) if not compare_digest(parsed[b'v'], server_sig): raise OperationFailure("Server returned an invalid signature.") # Depending on how it's configured, Cyrus SASL (which the server uses) # requires a third empty challenge. if not res['done']: cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']), ('payload', Binary(b''))]) res = sock_info.command(source, cmd) if not res['done']: raise OperationFailure('SASL conversation failed to complete.') def _password_digest(username, password): """Get a password digest to use for authentication. """ if not isinstance(password, string_type): raise TypeError("password must be an " "instance of %s" % (string_type.__name__,)) if len(password) == 0: raise ValueError("password can't be empty") if not isinstance(username, string_type): raise TypeError("password must be an " "instance of %s" % (string_type.__name__,)) md5hash = md5() data = "%s:mongo:%s" % (username, password) md5hash.update(data.encode('utf-8')) return _unicode(md5hash.hexdigest()) def _auth_key(nonce, username, password): """Get an auth key to use for authentication. """ digest = _password_digest(username, password) md5hash = md5() data = "%s%s%s" % (nonce, username, digest) md5hash.update(data.encode('utf-8')) return _unicode(md5hash.hexdigest()) def _authenticate_gssapi(credentials, sock_info): """Authenticate using GSSAPI. """ if not HAVE_KERBEROS: raise ConfigurationError('The "kerberos" module must be ' 'installed to use GSSAPI authentication.') try: username = credentials.username password = credentials.password props = credentials.mechanism_properties # Starting here and continuing through the while loop below - establish # the security context. See RFC 4752, Section 3.1, first paragraph. host = sock_info.address[0] if props.canonicalize_host_name: host = socket.getfqdn(host) service = props.service_name + '@' + host if props.service_realm is not None: service = service + '@' + props.service_realm if password is not None: if _USE_PRINCIPAL: # Note that, though we use unquote_plus for unquoting URI # options, we use quote here. Microsoft's UrlUnescape (used # by WinKerberos) doesn't support +. principal = ":".join((quote(username), quote(password))) result, ctx = kerberos.authGSSClientInit( service, principal, gssflags=kerberos.GSS_C_MUTUAL_FLAG) else: if '@' in username: user, domain = username.split('@', 1) else: user, domain = username, None result, ctx = kerberos.authGSSClientInit( service, gssflags=kerberos.GSS_C_MUTUAL_FLAG, user=user, domain=domain, password=password) else: result, ctx = kerberos.authGSSClientInit( service, gssflags=kerberos.GSS_C_MUTUAL_FLAG) if result != kerberos.AUTH_GSS_COMPLETE: raise OperationFailure('Kerberos context failed to initialize.') try: # pykerberos uses a weird mix of exceptions and return values # to indicate errors. # 0 == continue, 1 == complete, -1 == error # Only authGSSClientStep can return 0. if kerberos.authGSSClientStep(ctx, '') != 0: raise OperationFailure('Unknown kerberos ' 'failure in step function.') # Start a SASL conversation with mongod/s # Note: pykerberos deals with base64 encoded byte strings. # Since mongo accepts base64 strings as the payload we don't # have to use bson.binary.Binary. payload = kerberos.authGSSClientResponse(ctx) cmd = SON([('saslStart', 1), ('mechanism', 'GSSAPI'), ('payload', payload), ('autoAuthorize', 1)]) response = sock_info.command('$external', cmd) # Limit how many times we loop to catch protocol / library issues for _ in range(10): result = kerberos.authGSSClientStep(ctx, str(response['payload'])) if result == -1: raise OperationFailure('Unknown kerberos ' 'failure in step function.') payload = kerberos.authGSSClientResponse(ctx) or '' cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', payload)]) response = sock_info.command('$external', cmd) if result == kerberos.AUTH_GSS_COMPLETE: break else: raise OperationFailure('Kerberos ' 'authentication failed to complete.') # Once the security context is established actually authenticate. # See RFC 4752, Section 3.1, last two paragraphs. if kerberos.authGSSClientUnwrap(ctx, str(response['payload'])) != 1: raise OperationFailure('Unknown kerberos ' 'failure during GSS_Unwrap step.') if kerberos.authGSSClientWrap(ctx, kerberos.authGSSClientResponse(ctx), username) != 1: raise OperationFailure('Unknown kerberos ' 'failure during GSS_Wrap step.') payload = kerberos.authGSSClientResponse(ctx) cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', payload)]) sock_info.command('$external', cmd) finally: kerberos.authGSSClientClean(ctx) except kerberos.KrbError as exc: raise OperationFailure(str(exc)) def _authenticate_plain(credentials, sock_info): """Authenticate using SASL PLAIN (RFC 4616) """ source = credentials.source username = credentials.username password = credentials.password payload = ('\x00%s\x00%s' % (username, password)).encode('utf-8') cmd = SON([('saslStart', 1), ('mechanism', 'PLAIN'), ('payload', Binary(payload)), ('autoAuthorize', 1)]) sock_info.command(source, cmd) def _authenticate_cram_md5(credentials, sock_info): """Authenticate using CRAM-MD5 (RFC 2195) """ source = credentials.source username = credentials.username password = credentials.password # The password used as the mac key is the # same as what we use for MONGODB-CR passwd = _password_digest(username, password) cmd = SON([('saslStart', 1), ('mechanism', 'CRAM-MD5'), ('payload', Binary(b'')), ('autoAuthorize', 1)]) response = sock_info.command(source, cmd) # MD5 as implicit default digest for digestmod is deprecated # in python 3.4 mac = hmac.HMAC(key=passwd.encode('utf-8'), digestmod=md5) mac.update(response['payload']) challenge = username.encode('utf-8') + b' ' + b(mac.hexdigest()) cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', Binary(challenge))]) sock_info.command(source, cmd) def _authenticate_x509(credentials, sock_info): """Authenticate using MONGODB-X509. """ query = SON([('authenticate', 1), ('mechanism', 'MONGODB-X509')]) if credentials.username is not None: query['user'] = credentials.username elif sock_info.max_wire_version < 5: raise ConfigurationError( "A username is required for MONGODB-X509 authentication " "when connected to MongoDB versions older than 3.4.") sock_info.command('$external', query) def _authenticate_mongo_cr(credentials, sock_info): """Authenticate using MONGODB-CR. """ source = credentials.source username = credentials.username password = credentials.password # Get a nonce response = sock_info.command(source, {'getnonce': 1}) nonce = response['nonce'] key = _auth_key(nonce, username, password) # Actually authenticate query = SON([('authenticate', 1), ('user', username), ('nonce', nonce), ('key', key)]) sock_info.command(source, query) def _authenticate_default(credentials, sock_info): if sock_info.max_wire_version >= 3: return _authenticate_scram_sha1(credentials, sock_info) else: return _authenticate_mongo_cr(credentials, sock_info) _AUTH_MAP = { 'CRAM-MD5': _authenticate_cram_md5, 'GSSAPI': _authenticate_gssapi, 'MONGODB-CR': _authenticate_mongo_cr, 'MONGODB-X509': _authenticate_x509, 'PLAIN': _authenticate_plain, 'SCRAM-SHA-1': _authenticate_scram_sha1, 'DEFAULT': _authenticate_default, } def authenticate(credentials, sock_info): """Authenticate sock_info.""" mechanism = credentials.mechanism auth_func = _AUTH_MAP.get(mechanism) auth_func(credentials, sock_info) def logout(source, sock_info): """Log out from a database.""" sock_info.command(source, {'logout': 1}) pymongo-3.6.1/pymongo/monotonic.py0000644000076600000240000000211413245621354017505 0ustar shanestaff00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Time. Monotonic if possible. """ from __future__ import absolute_import __all__ = ['time'] try: # Patches standard time module. # From https://pypi.python.org/pypi/Monotime. import monotime except ImportError: pass try: # From https://pypi.python.org/pypi/monotonic. from monotonic import monotonic as time except ImportError: try: # Monotime or Python 3. from time import monotonic as time except ImportError: # Not monotonic. from time import time pymongo-3.6.1/pymongo/server.py0000644000076600000240000001347113245621354017016 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Communicate with one MongoDB server in a topology.""" import contextlib from datetime import datetime from pymongo.message import _convert_exception from pymongo.response import Response, ExhaustResponse from pymongo.server_type import SERVER_TYPE class Server(object): def __init__(self, server_description, pool, monitor, topology_id=None, listeners=None, events=None): """Represent one MongoDB server.""" self._description = server_description self._pool = pool self._monitor = monitor self._topology_id = topology_id self._publish = listeners is not None and listeners.enabled_for_server self._listener = listeners self._events = None if self._publish: self._events = events() def open(self): """Start monitoring, or restart after a fork. Multiple calls have no effect. """ self._monitor.open() def reset(self): """Clear the connection pool.""" self.pool.reset() def close(self): """Clear the connection pool and stop the monitor. Reconnect with open(). """ if self._publish: self._events.put((self._listener.publish_server_closed, (self._description.address, self._topology_id))) self._monitor.close() self._pool.reset() def request_check(self): """Check the server's state soon.""" self._monitor.request_check() def send_message_with_response( self, operation, set_slave_okay, all_credentials, listeners, exhaust=False): """Send a message to MongoDB and return a Response object. Can raise ConnectionFailure. :Parameters: - `operation`: A _Query or _GetMore object. - `set_slave_okay`: Pass to operation.get_message. - `all_credentials`: dict, maps auth source to MongoCredential. - `listeners`: Instance of _EventListeners or None. - `exhaust` (optional): If True, the socket used stays checked out. It is returned along with its Pool in the Response. """ with self.get_socket(all_credentials, exhaust) as sock_info: duration = None publish = listeners.enabled_for_commands if publish: start = datetime.now() use_find_cmd = operation.use_command(sock_info, exhaust) message = operation.get_message( set_slave_okay, sock_info, use_find_cmd) request_id, data, max_doc_size = self._split_message(message) if publish: encoding_duration = datetime.now() - start cmd, dbn = operation.as_command(sock_info) listeners.publish_command_start( cmd, dbn, request_id, sock_info.address) start = datetime.now() try: sock_info.send_message(data, max_doc_size) reply = sock_info.receive_message(request_id) except Exception as exc: if publish: duration = (datetime.now() - start) + encoding_duration failure = _convert_exception(exc) listeners.publish_command_failure( duration, failure, next(iter(cmd)), request_id, sock_info.address) raise if publish: duration = (datetime.now() - start) + encoding_duration if exhaust: return ExhaustResponse( data=reply, address=self._description.address, socket_info=sock_info, pool=self._pool, duration=duration, request_id=request_id, from_command=use_find_cmd) else: return Response( data=reply, address=self._description.address, duration=duration, request_id=request_id, from_command=use_find_cmd) @contextlib.contextmanager def get_socket(self, all_credentials, checkout=False): with self.pool.get_socket(all_credentials, checkout) as sock_info: yield sock_info @property def description(self): return self._description @description.setter def description(self, server_description): assert server_description.address == self._description.address self._description = server_description @property def pool(self): return self._pool def _split_message(self, message): """Return request_id, data, max_doc_size. :Parameters: - `message`: (request_id, data, max_doc_size) or (request_id, data) """ if len(message) == 3: return message else: # get_more and kill_cursors messages don't include BSON documents. request_id, data = message return request_id, data, 0 def __str__(self): d = self._description return '' % ( d.address[0], d.address[1], SERVER_TYPE._fields[d.server_type]) pymongo-3.6.1/pymongo/son_manipulator.py0000644000076600000240000001505313245621354020720 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """**DEPRECATED**: Manipulators that can edit SON objects as they enter and exit a database. The :class:`~pymongo.son_manipulator.SONManipulator` API has limitations as a technique for transforming your data. Instead, it is more flexible and straightforward to transform outgoing documents in your own code before passing them to PyMongo, and transform incoming documents after receiving them from PyMongo. SON Manipulators will be removed from PyMongo in 4.0. PyMongo does **not** apply SON manipulators to documents passed to the modern methods :meth:`~pymongo.collection.Collection.bulk_write`, :meth:`~pymongo.collection.Collection.insert_one`, :meth:`~pymongo.collection.Collection.insert_many`, :meth:`~pymongo.collection.Collection.update_one`, or :meth:`~pymongo.collection.Collection.update_many`. SON manipulators are **not** applied to documents returned by the modern methods :meth:`~pymongo.collection.Collection.find_one_and_delete`, :meth:`~pymongo.collection.Collection.find_one_and_replace`, and :meth:`~pymongo.collection.Collection.find_one_and_update`. """ from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import abc from bson.son import SON class SONManipulator(object): """A base son manipulator. This manipulator just saves and restores objects without changing them. """ def will_copy(self): """Will this SON manipulator make a copy of the incoming document? Derived classes that do need to make a copy should override this method, returning True instead of False. All non-copying manipulators will be applied first (so that the user's document will be updated appropriately), followed by copying manipulators. """ return False def transform_incoming(self, son, collection): """Manipulate an incoming SON object. :Parameters: - `son`: the SON object to be inserted into the database - `collection`: the collection the object is being inserted into """ if self.will_copy(): return SON(son) return son def transform_outgoing(self, son, collection): """Manipulate an outgoing SON object. :Parameters: - `son`: the SON object being retrieved from the database - `collection`: the collection this object was stored in """ if self.will_copy(): return SON(son) return son class ObjectIdInjector(SONManipulator): """A son manipulator that adds the _id field if it is missing. .. versionchanged:: 2.7 ObjectIdInjector is no longer used by PyMongo, but remains in this module for backwards compatibility. """ def transform_incoming(self, son, collection): """Add an _id field if it is missing. """ if not "_id" in son: son["_id"] = ObjectId() return son # This is now handled during BSON encoding (for performance reasons), # but I'm keeping this here as a reference for those implementing new # SONManipulators. class ObjectIdShuffler(SONManipulator): """A son manipulator that moves _id to the first position. """ def will_copy(self): """We need to copy to be sure that we are dealing with SON, not a dict. """ return True def transform_incoming(self, son, collection): """Move _id to the front if it's there. """ if not "_id" in son: return son transformed = SON({"_id": son["_id"]}) transformed.update(son) return transformed class NamespaceInjector(SONManipulator): """A son manipulator that adds the _ns field. """ def transform_incoming(self, son, collection): """Add the _ns field to the incoming object """ son["_ns"] = collection.name return son class AutoReference(SONManipulator): """Transparently reference and de-reference already saved embedded objects. This manipulator should probably only be used when the NamespaceInjector is also being used, otherwise it doesn't make too much sense - documents can only be auto-referenced if they have an *_ns* field. NOTE: this will behave poorly if you have a circular reference. TODO: this only works for documents that are in the same database. To fix this we'll need to add a DatabaseInjector that adds *_db* and then make use of the optional *database* support for DBRefs. """ def __init__(self, db): self.database = db def will_copy(self): """We need to copy so the user's document doesn't get transformed refs. """ return True def transform_incoming(self, son, collection): """Replace embedded documents with DBRefs. """ def transform_value(value): if isinstance(value, abc.MutableMapping): if "_id" in value and "_ns" in value: return DBRef(value["_ns"], transform_value(value["_id"])) else: return transform_dict(SON(value)) elif isinstance(value, list): return [transform_value(v) for v in value] return value def transform_dict(object): for (key, value) in object.items(): object[key] = transform_value(value) return object return transform_dict(SON(son)) def transform_outgoing(self, son, collection): """Replace DBRefs with embedded documents. """ def transform_value(value): if isinstance(value, DBRef): return self.database.dereference(value) elif isinstance(value, list): return [transform_value(v) for v in value] elif isinstance(value, abc.MutableMapping): return transform_dict(SON(value)) return value def transform_dict(object): for (key, value) in object.items(): object[key] = transform_value(value) return object return transform_dict(SON(son)) pymongo-3.6.1/pymongo/monitor.py0000644000076600000240000001506213245664476017211 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Class to monitor a MongoDB server on a background thread.""" import weakref from pymongo import common, periodic_executor from pymongo.errors import OperationFailure from pymongo.server_type import SERVER_TYPE from pymongo.monotonic import time as _time from pymongo.read_preferences import MovingAverage from pymongo.server_description import ServerDescription class Monitor(object): def __init__( self, server_description, topology, pool, topology_settings): """Class to monitor a MongoDB server on a background thread. Pass an initial ServerDescription, a Topology, a Pool, and TopologySettings. The Topology is weakly referenced. The Pool must be exclusive to this Monitor. """ self._server_description = server_description self._pool = pool self._settings = topology_settings self._avg_round_trip_time = MovingAverage() self._listeners = self._settings._pool_options.event_listeners pub = self._listeners is not None self._publish = pub and self._listeners.enabled_for_server_heartbeat # We strongly reference the executor and it weakly references us via # this closure. When the monitor is freed, stop the executor soon. def target(): monitor = self_ref() if monitor is None: return False # Stop the executor. Monitor._run(monitor) return True executor = periodic_executor.PeriodicExecutor( interval=self._settings.heartbeat_frequency, min_interval=common.MIN_HEARTBEAT_INTERVAL, target=target, name="pymongo_server_monitor_thread") self._executor = executor # Avoid cycles. When self or topology is freed, stop executor soon. self_ref = weakref.ref(self, executor.close) self._topology = weakref.proxy(topology, executor.close) def open(self): """Start monitoring, or restart after a fork. Multiple calls have no effect. """ self._executor.open() def close(self): """Close and stop monitoring. open() restarts the monitor after closing. """ self._executor.close() # Increment the pool_id and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. self._pool.reset() def join(self, timeout=None): self._executor.join(timeout) def request_check(self): """If the monitor is sleeping, wake and check the server soon.""" self._executor.wake() def _run(self): try: self._server_description = self._check_with_retry() self._topology.on_change(self._server_description) except ReferenceError: # Topology was garbage-collected. self.close() def _check_with_retry(self): """Call ismaster once or twice. Reset server's pool on error. Returns a ServerDescription. """ # According to the spec, if an ismaster call fails we reset the # server's pool. If a server was once connected, change its type # to Unknown only after retrying once. address = self._server_description.address retry = True if self._server_description.server_type == SERVER_TYPE.Unknown: retry = False start = _time() try: return self._check_once() except ReferenceError: raise except Exception as error: error_time = _time() - start if self._publish: self._listeners.publish_server_heartbeat_failed( address, error_time, error) self._topology.reset_pool(address) default = ServerDescription(address, error=error) if not retry: self._avg_round_trip_time.reset() # Server type defaults to Unknown. return default # Try a second and final time. If it fails return original error. # Always send metadata: this is a new connection. start = _time() try: return self._check_once() except ReferenceError: raise except Exception as error: error_time = _time() - start if self._publish: self._listeners.publish_server_heartbeat_failed( address, error_time, error) self._avg_round_trip_time.reset() return default def _check_once(self): """A single attempt to call ismaster. Returns a ServerDescription, or raises an exception. """ address = self._server_description.address if self._publish: self._listeners.publish_server_heartbeat_started(address) with self._pool.get_socket({}) as sock_info: response, round_trip_time = self._check_with_socket(sock_info) self._avg_round_trip_time.add_sample(round_trip_time) sd = ServerDescription( address=address, ismaster=response, round_trip_time=self._avg_round_trip_time.get()) if self._publish: self._listeners.publish_server_heartbeat_succeeded( address, round_trip_time, response) return sd def _check_with_socket(self, sock_info): """Return (IsMaster, round_trip_time). Can raise ConnectionFailure or OperationFailure. """ start = _time() try: return (sock_info.ismaster(self._pool.opts.metadata, self._topology.max_cluster_time()), _time() - start) except OperationFailure as exc: # Update max cluster time even when isMaster fails. self._topology.receive_cluster_time( exc.details.get('$clusterTime')) raise pymongo-3.6.1/pymongo/results.py0000644000076600000240000001721213245621354017206 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Result class definitions.""" from pymongo.errors import InvalidOperation class _WriteResult(object): """Base class for write result classes.""" __slots__ = ("__acknowledged",) def __init__(self, acknowledged): self.__acknowledged = acknowledged def _raise_if_unacknowledged(self, property_name): """Raise an exception on property access if unacknowledged.""" if not self.__acknowledged: raise InvalidOperation("A value for %s is not available when " "the write is unacknowledged. Check the " "acknowledged attribute to avoid this " "error." % (property_name,)) @property def acknowledged(self): """Is this the result of an acknowledged write operation? The :attr:`acknowledged` attribute will be ``False`` when using ``WriteConcern(w=0)``, otherwise ``True``. .. note:: If the :attr:`acknowledged` attribute is ``False`` all other attibutes of this class will raise :class:`~pymongo.errors.InvalidOperation` when accessed. Values for other attributes cannot be determined if the write operation was unacknowledged. .. seealso:: :class:`~pymongo.write_concern.WriteConcern` """ return self.__acknowledged class InsertOneResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.insert_one`. """ __slots__ = ("__inserted_id", "__acknowledged") def __init__(self, inserted_id, acknowledged): self.__inserted_id = inserted_id super(InsertOneResult, self).__init__(acknowledged) @property def inserted_id(self): """The inserted document's _id.""" return self.__inserted_id class InsertManyResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.insert_many`. """ __slots__ = ("__inserted_ids", "__acknowledged") def __init__(self, inserted_ids, acknowledged): self.__inserted_ids = inserted_ids super(InsertManyResult, self).__init__(acknowledged) @property def inserted_ids(self): """A list of _ids of the inserted documents, in the order provided. .. note:: If ``False`` is passed for the `ordered` parameter to :meth:`~pymongo.collection.Collection.insert_many` the server may have inserted the documents in a different order than what is presented here. """ return self.__inserted_ids class UpdateResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.update_one`, :meth:`~pymongo.collection.Collection.update_many`, and :meth:`~pymongo.collection.Collection.replace_one`. """ __slots__ = ("__raw_result", "__acknowledged") def __init__(self, raw_result, acknowledged): self.__raw_result = raw_result super(UpdateResult, self).__init__(acknowledged) @property def raw_result(self): """The raw result document returned by the server.""" return self.__raw_result @property def matched_count(self): """The number of documents matched for this update.""" self._raise_if_unacknowledged("matched_count") if self.upserted_id is not None: return 0 return self.__raw_result.get("n", 0) @property def modified_count(self): """The number of documents modified. .. note:: modified_count is only reported by MongoDB 2.6 and later. When connected to an earlier server version, or in certain mixed version sharding configurations, this attribute will be set to ``None``. """ self._raise_if_unacknowledged("modified_count") return self.__raw_result.get("nModified") @property def upserted_id(self): """The _id of the inserted document if an upsert took place. Otherwise ``None``. """ self._raise_if_unacknowledged("upserted_id") return self.__raw_result.get("upserted") class DeleteResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.delete_one` and :meth:`~pymongo.collection.Collection.delete_many`""" __slots__ = ("__raw_result", "__acknowledged") def __init__(self, raw_result, acknowledged): self.__raw_result = raw_result super(DeleteResult, self).__init__(acknowledged) @property def raw_result(self): """The raw result document returned by the server.""" return self.__raw_result @property def deleted_count(self): """The number of documents deleted.""" self._raise_if_unacknowledged("deleted_count") return self.__raw_result.get("n", 0) class BulkWriteResult(_WriteResult): """An object wrapper for bulk API write results.""" __slots__ = ("__bulk_api_result", "__acknowledged") def __init__(self, bulk_api_result, acknowledged): """Create a BulkWriteResult instance. :Parameters: - `bulk_api_result`: A result dict from the bulk API - `acknowledged`: Was this write result acknowledged? If ``False`` then all properties of this object will raise :exc:`~pymongo.errors.InvalidOperation`. """ self.__bulk_api_result = bulk_api_result super(BulkWriteResult, self).__init__(acknowledged) @property def bulk_api_result(self): """The raw bulk API result.""" return self.__bulk_api_result @property def inserted_count(self): """The number of documents inserted.""" self._raise_if_unacknowledged("inserted_count") return self.__bulk_api_result.get("nInserted") @property def matched_count(self): """The number of documents matched for an update.""" self._raise_if_unacknowledged("matched_count") return self.__bulk_api_result.get("nMatched") @property def modified_count(self): """The number of documents modified. .. note:: modified_count is only reported by MongoDB 2.6 and later. When connected to an earlier server version, or in certain mixed version sharding configurations, this attribute will be set to ``None``. """ self._raise_if_unacknowledged("modified_count") return self.__bulk_api_result.get("nModified") @property def deleted_count(self): """The number of documents deleted.""" self._raise_if_unacknowledged("deleted_count") return self.__bulk_api_result.get("nRemoved") @property def upserted_count(self): """The number of documents upserted.""" self._raise_if_unacknowledged("upserted_count") return self.__bulk_api_result.get("nUpserted") @property def upserted_ids(self): """A map of operation index to the _id of the upserted document.""" self._raise_if_unacknowledged("upserted_ids") if self.__bulk_api_result: return dict((upsert["index"], upsert["_id"]) for upsert in self.bulk_api_result["upserted"]) pymongo-3.6.1/pymongo/bulk.py0000644000076600000240000005561613245621354016454 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """The bulk write operations interface. .. versionadded:: 2.7 """ from itertools import islice from bson.objectid import ObjectId from bson.raw_bson import RawBSONDocument from bson.son import SON from pymongo.common import (validate_is_mapping, validate_is_document_type, validate_ok_for_replace, validate_ok_for_update) from pymongo.collation import validate_collation_or_none from pymongo.errors import (BulkWriteError, ConfigurationError, ConnectionFailure, InvalidOperation, OperationFailure, ServerSelectionTimeoutError) from pymongo.message import (_INSERT, _UPDATE, _DELETE, _do_batched_insert, _do_batched_write_command, _randint, _BulkWriteContext) from pymongo.write_concern import WriteConcern _DELETE_ALL = 0 _DELETE_ONE = 1 # For backwards compatibility. See MongoDB src/mongo/base/error_codes.err _BAD_VALUE = 2 _UNKNOWN_ERROR = 8 _WRITE_CONCERN_ERROR = 64 _COMMANDS = ('insert', 'update', 'delete') # These string literals are used when we create fake server return # documents client side. We use unicode literals in python 2.x to # match the actual return values from the server. _UOP = u"op" class _Run(object): """Represents a batch of write operations. """ def __init__(self, op_type): """Initialize a new Run object. """ self.op_type = op_type self.index_map = [] self.ops = [] self.idx_offset = 0 def index(self, idx): """Get the original index of an operation in this run. :Parameters: - `idx`: The Run index that maps to the original index. """ return self.index_map[idx] def add(self, original_index, operation): """Add an operation to this Run instance. :Parameters: - `original_index`: The original index of this operation within a larger bulk operation. - `operation`: The operation document. """ self.index_map.append(original_index) self.ops.append(operation) def _merge_command(run, full_result, results): """Merge a group of results from write commands into the full result. """ for offset, result in results: affected = result.get("n", 0) if run.op_type == _INSERT: full_result["nInserted"] += affected elif run.op_type == _DELETE: full_result["nRemoved"] += affected elif run.op_type == _UPDATE: upserted = result.get("upserted") if upserted: n_upserted = len(upserted) for doc in upserted: doc["index"] = run.index(doc["index"] + offset) full_result["upserted"].extend(upserted) full_result["nUpserted"] += n_upserted full_result["nMatched"] += (affected - n_upserted) else: full_result["nMatched"] += affected full_result["nModified"] += result["nModified"] write_errors = result.get("writeErrors") if write_errors: for doc in write_errors: # Leave the server response intact for APM. replacement = doc.copy() idx = doc["index"] + offset replacement["index"] = run.index(idx) # Add the failed operation to the error document. replacement[_UOP] = run.ops[idx] full_result["writeErrors"].append(replacement) wc_error = result.get("writeConcernError") if wc_error: full_result["writeConcernErrors"].append(wc_error) class _Bulk(object): """The private guts of the bulk write API. """ def __init__(self, collection, ordered, bypass_document_validation): """Initialize a _Bulk instance. """ self.collection = collection.with_options( codec_options=collection.codec_options._replace( unicode_decode_error_handler='replace', document_class=dict)) self.ordered = ordered self.ops = [] self.name = "%s.%s" % (collection.database.name, collection.name) self.namespace = collection.database.name + '.$cmd' self.executed = False self.bypass_doc_val = bypass_document_validation self.uses_collation = False self.uses_array_filters = False self.is_retryable = True self.retrying = False # Extra state so that we know where to pick up on a retry attempt. self.current_run = None def add_insert(self, document): """Add an insert document to the list of ops. """ validate_is_document_type("document", document) # Generate ObjectId client side. if not (isinstance(document, RawBSONDocument) or '_id' in document): document['_id'] = ObjectId() self.ops.append((_INSERT, document)) def add_update(self, selector, update, multi=False, upsert=False, collation=None, array_filters=None): """Create an update document and add it to the list of ops. """ validate_ok_for_update(update) cmd = SON([('q', selector), ('u', update), ('multi', multi), ('upsert', upsert)]) collation = validate_collation_or_none(collation) if collation is not None: self.uses_collation = True cmd['collation'] = collation if array_filters is not None: self.uses_array_filters = True cmd['arrayFilters'] = array_filters if multi: # A bulk_write containing an update_many is not retryable. self.is_retryable = False self.ops.append((_UPDATE, cmd)) def add_replace(self, selector, replacement, upsert=False, collation=None): """Create a replace document and add it to the list of ops. """ validate_ok_for_replace(replacement) cmd = SON([('q', selector), ('u', replacement), ('multi', False), ('upsert', upsert)]) collation = validate_collation_or_none(collation) if collation is not None: self.uses_collation = True cmd['collation'] = collation self.ops.append((_UPDATE, cmd)) def add_delete(self, selector, limit, collation=None): """Create a delete document and add it to the list of ops. """ cmd = SON([('q', selector), ('limit', limit)]) collation = validate_collation_or_none(collation) if collation is not None: self.uses_collation = True cmd['collation'] = collation if limit == _DELETE_ALL: # A bulk_write containing a delete_many is not retryable. self.is_retryable = False self.ops.append((_DELETE, cmd)) def gen_ordered(self): """Generate batches of operations, batched by type of operation, in the order **provided**. """ run = None for idx, (op_type, operation) in enumerate(self.ops): if run is None: run = _Run(op_type) elif run.op_type != op_type: yield run run = _Run(op_type) run.add(idx, operation) yield run def gen_unordered(self): """Generate batches of operations, batched by type of operation, in arbitrary order. """ operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] for idx, (op_type, operation) in enumerate(self.ops): operations[op_type].add(idx, operation) for run in operations: if run.ops: yield run def _execute_command(self, generator, write_concern, session, sock_info, op_id, retryable, full_result): if sock_info.max_wire_version < 5 and self.uses_collation: raise ConfigurationError( 'Must be connected to MongoDB 3.4+ to use a collation.') if sock_info.max_wire_version < 6 and self.uses_array_filters: raise ConfigurationError( 'Must be connected to MongoDB 3.6+ to use arrayFilters.') db_name = self.collection.database.name client = self.collection.database.client listeners = client._event_listeners if not self.current_run: self.current_run = next(generator) run = self.current_run # sock_info.command validates the session, but we use # sock_info.write_command. sock_info.validate_session(client, session) while run: cmd = SON([(_COMMANDS[run.op_type], self.collection.name), ('ordered', self.ordered)]) if write_concern.document: cmd['writeConcern'] = write_concern.document if self.bypass_doc_val and sock_info.max_wire_version >= 4: cmd['bypassDocumentValidation'] = True if session: cmd['lsid'] = session._use_lsid() bwc = _BulkWriteContext(db_name, cmd, sock_info, op_id, listeners, session) results = [] while run.idx_offset < len(run.ops): if session and retryable: cmd['txnNumber'] = session._transaction_id() sock_info.send_cluster_time(cmd, session, client) check_keys = run.op_type == _INSERT ops = islice(run.ops, run.idx_offset, None) # Run as many ops as possible. request_id, msg, to_send = _do_batched_write_command( self.namespace, run.op_type, cmd, ops, check_keys, self.collection.codec_options, bwc) if not to_send: raise InvalidOperation("cannot do an empty bulk write") result = bwc.write_command(request_id, msg, to_send) client._receive_cluster_time(result, session) results.append((run.idx_offset, result)) # We're no longer in a retry once a command succeeds. self.retrying = False if self.ordered and "writeErrors" in result: break run.idx_offset += len(to_send) _merge_command(run, full_result, results) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result['writeErrors']: break # Reset our state self.current_run = run = next(generator, None) def execute_command(self, generator, write_concern, session): """Execute using write commands. """ # nModified is only reported for write commands, not legacy ops. full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], } op_id = _randint() def retryable_bulk(session, sock_info, retryable): self._execute_command( generator, write_concern, session, sock_info, op_id, retryable, full_result) client = self.collection.database.client with client._tmp_session(session) as s: client._retry_with_session( self.is_retryable, retryable_bulk, s, self) if full_result["writeErrors"] or full_result["writeConcernErrors"]: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result def execute_insert_no_results(self, sock_info, run, op_id, acknowledged): """Execute insert, returning no results. """ command = SON([('insert', self.collection.name), ('ordered', self.ordered)]) concern = {'w': int(self.ordered)} command['writeConcern'] = concern if self.bypass_doc_val and sock_info.max_wire_version >= 4: command['bypassDocumentValidation'] = True db = self.collection.database bwc = _BulkWriteContext( db.name, command, sock_info, op_id, db.client._event_listeners, session=None) # Legacy batched OP_INSERT. _do_batched_insert( self.collection.full_name, run.ops, True, acknowledged, concern, not self.ordered, self.collection.codec_options, bwc) def execute_no_results(self, sock_info, generator): """Execute all operations, returning no results (w=0). """ if self.uses_collation: raise ConfigurationError( 'Collation is unsupported for unacknowledged writes.') if self.uses_array_filters: raise ConfigurationError( 'arrayFilters is unsupported for unacknowledged writes.') # Cannot have both unacknowledged write and bypass document validation. if self.bypass_doc_val and sock_info.max_wire_version >= 4: raise OperationFailure("Cannot set bypass_document_validation with" " unacknowledged write concern") coll = self.collection # If ordered is True we have to send GLE or use write # commands so we can abort on the first error. write_concern = WriteConcern(w=int(self.ordered)) op_id = _randint() next_run = next(generator) while next_run: # An ordered bulk write needs to send acknowledged writes to short # circuit the next run. However, the final message on the final # run can be unacknowledged. run = next_run next_run = next(generator, None) needs_ack = self.ordered and next_run is not None try: if run.op_type == _INSERT: self.execute_insert_no_results( sock_info, run, op_id, needs_ack) elif run.op_type == _UPDATE: for operation in run.ops: doc = operation['u'] check_keys = True if doc and next(iter(doc)).startswith('$'): check_keys = False coll._update( sock_info, operation['q'], doc, operation['upsert'], check_keys, operation['multi'], write_concern=write_concern, op_id=op_id, ordered=self.ordered, bypass_doc_val=self.bypass_doc_val) else: for operation in run.ops: coll._delete(sock_info, operation['q'], not operation['limit'], write_concern, op_id, self.ordered) except OperationFailure: if self.ordered: break def execute(self, write_concern, session): """Execute operations. """ if not self.ops: raise InvalidOperation('No operations to execute') if self.executed: raise InvalidOperation('Bulk operations can ' 'only be executed once.') self.executed = True write_concern = (WriteConcern(**write_concern) if write_concern else self.collection.write_concern) if self.ordered: generator = self.gen_ordered() else: generator = self.gen_unordered() client = self.collection.database.client if not write_concern.acknowledged: with client._socket_for_writes() as sock_info: self.execute_no_results(sock_info, generator) else: return self.execute_command(generator, write_concern, session) class BulkUpsertOperation(object): """An interface for adding upsert operations. """ __slots__ = ('__selector', '__bulk', '__collation') def __init__(self, selector, bulk, collation): self.__selector = selector self.__bulk = bulk self.__collation = collation def update_one(self, update): """Update one document matching the selector. :Parameters: - `update` (dict): the update operations to apply """ self.__bulk.add_update(self.__selector, update, multi=False, upsert=True, collation=self.__collation) def update(self, update): """Update all documents matching the selector. :Parameters: - `update` (dict): the update operations to apply """ self.__bulk.add_update(self.__selector, update, multi=True, upsert=True, collation=self.__collation) def replace_one(self, replacement): """Replace one entire document matching the selector criteria. :Parameters: - `replacement` (dict): the replacement document """ self.__bulk.add_replace(self.__selector, replacement, upsert=True, collation=self.__collation) class BulkWriteOperation(object): """An interface for adding update or remove operations. """ __slots__ = ('__selector', '__bulk', '__collation') def __init__(self, selector, bulk, collation): self.__selector = selector self.__bulk = bulk self.__collation = collation def update_one(self, update): """Update one document matching the selector criteria. :Parameters: - `update` (dict): the update operations to apply """ self.__bulk.add_update(self.__selector, update, multi=False, collation=self.__collation) def update(self, update): """Update all documents matching the selector criteria. :Parameters: - `update` (dict): the update operations to apply """ self.__bulk.add_update(self.__selector, update, multi=True, collation=self.__collation) def replace_one(self, replacement): """Replace one entire document matching the selector criteria. :Parameters: - `replacement` (dict): the replacement document """ self.__bulk.add_replace(self.__selector, replacement, collation=self.__collation) def remove_one(self): """Remove a single document matching the selector criteria. """ self.__bulk.add_delete(self.__selector, _DELETE_ONE, collation=self.__collation) def remove(self): """Remove all documents matching the selector criteria. """ self.__bulk.add_delete(self.__selector, _DELETE_ALL, collation=self.__collation) def upsert(self): """Specify that all chained update operations should be upserts. :Returns: - A :class:`BulkUpsertOperation` instance, used to add update operations to this bulk operation. """ return BulkUpsertOperation(self.__selector, self.__bulk, self.__collation) class BulkOperationBuilder(object): """**DEPRECATED**: An interface for executing a batch of write operations. """ __slots__ = '__bulk' def __init__(self, collection, ordered=True, bypass_document_validation=False): """**DEPRECATED**: Initialize a new BulkOperationBuilder instance. :Parameters: - `collection`: A :class:`~pymongo.collection.Collection` instance. - `ordered` (optional): If ``True`` all operations will be executed serially, in the order provided, and the entire execution will abort on the first error. If ``False`` operations will be executed in arbitrary order (possibly in parallel on the server), reporting any errors that occurred after attempting all operations. Defaults to ``True``. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.5 Deprecated. Use :meth:`~pymongo.collection.Collection.bulk_write` instead. .. versionchanged:: 3.2 Added bypass_document_validation support """ self.__bulk = _Bulk(collection, ordered, bypass_document_validation) def find(self, selector, collation=None): """Specify selection criteria for bulk operations. :Parameters: - `selector` (dict): the selection criteria for update and remove operations. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. :Returns: - A :class:`BulkWriteOperation` instance, used to add update and remove operations to this bulk operation. .. versionchanged:: 3.4 Added the `collation` option. """ validate_is_mapping("selector", selector) return BulkWriteOperation(selector, self.__bulk, collation) def insert(self, document): """Insert a single document. :Parameters: - `document` (dict): the document to insert .. seealso:: :ref:`writes-and-ids` """ self.__bulk.add_insert(document) def execute(self, write_concern=None): """Execute all provided operations. :Parameters: - write_concern (optional): the write concern for this bulk execution. """ if write_concern is not None: validate_is_mapping("write_concern", write_concern) return self.__bulk.execute(write_concern, session=None) pymongo-3.6.1/pymongo/periodic_executor.py0000644000076600000240000001263713245621354021227 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Run a target function on a background thread.""" import atexit import threading import time import weakref from pymongo.monotonic import time as _time class PeriodicExecutor(object): def __init__(self, interval, min_interval, target, name=None): """"Run a target function periodically on a background thread. If the target's return value is false, the executor stops. :Parameters: - `interval`: Seconds between calls to `target`. - `min_interval`: Minimum seconds between calls if `wake` is called very often. - `target`: A function. - `name`: A name to give the underlying thread. """ # threading.Event and its internal condition variable are expensive # in Python 2, see PYTHON-983. Use a boolean to know when to wake. # The executor's design is constrained by several Python issues, see # "periodic_executor.rst" in this repository. self._event = False self._interval = interval self._min_interval = min_interval self._target = target self._stopped = False self._thread = None self._name = name self._thread_will_exit = False self._lock = threading.Lock() def open(self): """Start. Multiple calls have no effect. Not safe to call from multiple threads at once. """ with self._lock: if self._thread_will_exit: # If the background thread has read self._stopped as True # there is a chance that it has not yet exited. The call to # join should not block indefinitely because there is no # other work done outside the while loop in self._run. try: self._thread.join() except ReferenceError: # Thread terminated. pass self._thread_will_exit = False self._stopped = False started = False try: started = self._thread and self._thread.is_alive() except ReferenceError: # Thread terminated. pass if not started: thread = threading.Thread(target=self._run, name=self._name) thread.daemon = True self._thread = weakref.proxy(thread) _register_executor(self) thread.start() def close(self, dummy=None): """Stop. To restart, call open(). The dummy parameter allows an executor's close method to be a weakref callback; see monitor.py. """ self._stopped = True def join(self, timeout=None): if self._thread is not None: try: self._thread.join(timeout) except (ReferenceError, RuntimeError): # Thread already terminated, or not yet started. pass def wake(self): """Execute the target function soon.""" self._event = True def __should_stop(self): with self._lock: if self._stopped: self._thread_will_exit = True return True return False def _run(self): while not self.__should_stop(): try: if not self._target(): self._stopped = True break except: with self._lock: self._stopped = True self._thread_will_exit = True raise deadline = _time() + self._interval while not self._stopped and _time() < deadline: time.sleep(self._min_interval) if self._event: break # Early wake. self._event = False # _EXECUTORS has a weakref to each running PeriodicExecutor. Once started, # an executor is kept alive by a strong reference from its thread and perhaps # from other objects. When the thread dies and all other referrers are freed, # the executor is freed and removed from _EXECUTORS. If any threads are # running when the interpreter begins to shut down, we try to halt and join # them to avoid spurious errors. _EXECUTORS = set() def _register_executor(executor): ref = weakref.ref(executor, _on_executor_deleted) _EXECUTORS.add(ref) def _on_executor_deleted(ref): _EXECUTORS.remove(ref) def _shutdown_executors(): if _EXECUTORS is None: return # Copy the set. Stopping threads has the side effect of removing executors. executors = list(_EXECUTORS) # First signal all executors to close... for ref in executors: executor = ref() if executor: executor.close() # ...then try to join them. for ref in executors: executor = ref() if executor: executor.join(1) executor = None atexit.register(_shutdown_executors) pymongo-3.6.1/pymongo/monitoring.py0000644000076600000240000007752413245621354017706 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Tools to monitor driver events. .. versionadded:: 3.1 Use :func:`register` to register global listeners for specific events. Listeners must inherit from one of the abstract classes below and implement the correct functions for that class. For example, a simple command logger might be implemented like this:: import logging from pymongo import monitoring class CommandLogger(monitoring.CommandListener): def started(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} started on server " "{0.connection_id}".format(event)) def succeeded(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} on server {0.connection_id} " "succeeded in {0.duration_micros} " "microseconds".format(event)) def failed(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} on server {0.connection_id} " "failed in {0.duration_micros} " "microseconds".format(event)) monitoring.register(CommandLogger()) Server discovery and monitoring events are also available. For example:: class ServerLogger(monitoring.ServerListener): def opened(self, event): logging.info("Server {0.server_address} added to topology " "{0.topology_id}".format(event)) def description_changed(self, event): previous_server_type = event.previous_description.server_type new_server_type = event.new_description.server_type if new_server_type != previous_server_type: # server_type_name was added in PyMongo 3.4 logging.info( "Server {0.server_address} changed type from " "{0.previous_description.server_type_name} to " "{0.new_description.server_type_name}".format(event)) def closed(self, event): logging.warning("Server {0.server_address} removed from topology " "{0.topology_id}".format(event)) class HeartbeatLogger(monitoring.ServerHeartbeatListener): def started(self, event): logging.info("Heartbeat sent to server " "{0.connection_id}".format(event)) def succeeded(self, event): # The reply.document attribute was added in PyMongo 3.4. logging.info("Heartbeat to server {0.connection_id} " "succeeded with reply " "{0.reply.document}".format(event)) def failed(self, event): logging.warning("Heartbeat to server {0.connection_id} " "failed with error {0.reply}".format(event)) class TopologyLogger(monitoring.TopologyListener): def opened(self, event): logging.info("Topology with id {0.topology_id} " "opened".format(event)) def description_changed(self, event): logging.info("Topology description updated for " "topology id {0.topology_id}".format(event)) previous_topology_type = event.previous_description.topology_type new_topology_type = event.new_description.topology_type if new_topology_type != previous_topology_type: # topology_type_name was added in PyMongo 3.4 logging.info( "Topology {0.topology_id} changed type from " "{0.previous_description.topology_type_name} to " "{0.new_description.topology_type_name}".format(event)) # The has_writable_server and has_readable_server methods # were added in PyMongo 3.4. if not event.new_description.has_writable_server(): logging.warning("No writable servers available.") if not event.new_description.has_readable_server(): logging.warning("No readable servers available.") def closed(self, event): logging.info("Topology with id {0.topology_id} " "closed".format(event)) Event listeners can also be registered per instance of :class:`~pymongo.mongo_client.MongoClient`:: client = MongoClient(event_listeners=[CommandLogger()]) Note that previously registered global listeners are automatically included when configuring per client event listeners. Registering a new global listener will not add that listener to existing client instances. .. note:: Events are delivered **synchronously**. Application threads block waiting for event handlers (e.g. :meth:`~CommandListener.started`) to return. Care must be taken to ensure that your event handlers are efficient enough to not adversely affect overall application performance. .. warning:: The command documents published through this API are *not* copies. If you intend to modify them in any way you must copy them in your event handler first. """ import sys import traceback from collections import namedtuple from bson.py3compat import abc from pymongo.helpers import _handle_exception _Listeners = namedtuple('Listeners', ('command_listeners', 'server_listeners', 'server_heartbeat_listeners', 'topology_listeners')) _LISTENERS = _Listeners([], [], [], []) class _EventListener(object): """Abstract base class for all event listeners.""" class CommandListener(_EventListener): """Abstract base class for command listeners. Handles `CommandStartedEvent`, `CommandSucceededEvent`, and `CommandFailedEvent`.""" def started(self, event): """Abstract method to handle a `CommandStartedEvent`. :Parameters: - `event`: An instance of :class:`CommandStartedEvent`. """ raise NotImplementedError def succeeded(self, event): """Abstract method to handle a `CommandSucceededEvent`. :Parameters: - `event`: An instance of :class:`CommandSucceededEvent`. """ raise NotImplementedError def failed(self, event): """Abstract method to handle a `CommandFailedEvent`. :Parameters: - `event`: An instance of :class:`CommandFailedEvent`. """ raise NotImplementedError class ServerHeartbeatListener(_EventListener): """Abstract base class for server heartbeat listeners. Handles `ServerHeartbeatStartedEvent`, `ServerHeartbeatSucceededEvent`, and `ServerHeartbeatFailedEvent`. .. versionadded:: 3.3 """ def started(self, event): """Abstract method to handle a `ServerHeartbeatStartedEvent`. :Parameters: - `event`: An instance of :class:`ServerHeartbeatStartedEvent`. """ raise NotImplementedError def succeeded(self, event): """Abstract method to handle a `ServerHeartbeatSucceededEvent`. :Parameters: - `event`: An instance of :class:`ServerHeartbeatSucceededEvent`. """ raise NotImplementedError def failed(self, event): """Abstract method to handle a `ServerHeartbeatFailedEvent`. :Parameters: - `event`: An instance of :class:`ServerHeartbeatFailedEvent`. """ raise NotImplementedError class TopologyListener(_EventListener): """Abstract base class for topology monitoring listeners. Handles `TopologyOpenedEvent`, `TopologyDescriptionChangedEvent`, and `TopologyClosedEvent`. .. versionadded:: 3.3 """ def opened(self, event): """Abstract method to handle a `TopologyOpenedEvent`. :Parameters: - `event`: An instance of :class:`TopologyOpenedEvent`. """ raise NotImplementedError def description_changed(self, event): """Abstract method to handle a `TopologyDescriptionChangedEvent`. :Parameters: - `event`: An instance of :class:`TopologyDescriptionChangedEvent`. """ raise NotImplementedError def closed(self, event): """Abstract method to handle a `TopologyClosedEvent`. :Parameters: - `event`: An instance of :class:`TopologyClosedEvent`. """ raise NotImplementedError class ServerListener(_EventListener): """Abstract base class for server listeners. Handles `ServerOpeningEvent`, `ServerDescriptionChangedEvent`, and `ServerClosedEvent`. .. versionadded:: 3.3 """ def opened(self, event): """Abstract method to handle a `ServerOpeningEvent`. :Parameters: - `event`: An instance of :class:`ServerOpeningEvent`. """ raise NotImplementedError def description_changed(self, event): """Abstract method to handle a `ServerDescriptionChangedEvent`. :Parameters: - `event`: An instance of :class:`ServerDescriptionChangedEvent`. """ raise NotImplementedError def closed(self, event): """Abstract method to handle a `ServerClosedEvent`. :Parameters: - `event`: An instance of :class:`ServerClosedEvent`. """ raise NotImplementedError def _to_micros(dur): """Convert duration 'dur' to microseconds.""" if hasattr(dur, 'total_seconds'): return int(dur.total_seconds() * 10e5) # Python 2.6 return dur.microseconds + (dur.seconds + dur.days * 24 * 3600) * 1000000 def _validate_event_listeners(option, listeners): """Validate event listeners""" if not isinstance(listeners, abc.Sequence): raise TypeError("%s must be a list or tuple" % (option,)) for listener in listeners: if not isinstance(listener, _EventListener): raise TypeError("Listeners for %s must be either a " "CommandListener, ServerHeartbeatListener, " "ServerListener, or TopologyListener." % (option,)) return listeners def register(listener): """Register a global event listener. :Parameters: - `listener`: A subclasses of :class:`CommandListener`, :class:`ServerHeartbeatListener`, :class:`ServerListener`, or :class:`TopologyListener`. """ if not isinstance(listener, _EventListener): raise TypeError("Listeners for %s must be either a " "CommandListener, ServerHeartbeatListener, " "ServerListener, or TopologyListener." % (listener,)) if isinstance(listener, CommandListener): _LISTENERS.command_listeners.append(listener) if isinstance(listener, ServerHeartbeatListener): _LISTENERS.server_heartbeat_listeners.append(listener) if isinstance(listener, ServerListener): _LISTENERS.server_listeners.append(listener) if isinstance(listener, TopologyListener): _LISTENERS.topology_listeners.append(listener) # Note - to avoid bugs from forgetting which if these is all lowercase and # which are camelCase, and at the same time avoid having to add a test for # every command, use all lowercase here and test against command_name.lower(). _SENSITIVE_COMMANDS = set( ["authenticate", "saslstart", "saslcontinue", "getnonce", "createuser", "updateuser", "copydbgetnonce", "copydbsaslstart", "copydb"]) class _CommandEvent(object): """Base class for command events.""" __slots__ = ("__cmd_name", "__rqst_id", "__conn_id", "__op_id") def __init__(self, command_name, request_id, connection_id, operation_id): self.__cmd_name = command_name self.__rqst_id = request_id self.__conn_id = connection_id self.__op_id = operation_id @property def command_name(self): """The command name.""" return self.__cmd_name @property def request_id(self): """The request id for this operation.""" return self.__rqst_id @property def connection_id(self): """The address (host, port) of the server this command was sent to.""" return self.__conn_id @property def operation_id(self): """An id for this series of events or None.""" return self.__op_id class CommandStartedEvent(_CommandEvent): """Event published when a command starts. :Parameters: - `command`: The command document. - `database_name`: The name of the database this command was run against. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `operation_id`: An optional identifier for a series of related events. """ __slots__ = ("__cmd", "__db") def __init__(self, command, database_name, *args): if not command: raise ValueError("%r is not a valid command" % (command,)) # Command name must be first key. command_name = next(iter(command)) super(CommandStartedEvent, self).__init__(command_name, *args) if command_name.lower() in _SENSITIVE_COMMANDS: self.__cmd = {} else: self.__cmd = command self.__db = database_name @property def command(self): """The command document.""" return self.__cmd @property def database_name(self): """The name of the database this command was run against.""" return self.__db class CommandSucceededEvent(_CommandEvent): """Event published when a command succeeds. :Parameters: - `duration`: The command duration as a datetime.timedelta. - `reply`: The server reply document. - `command_name`: The command name. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `operation_id`: An optional identifier for a series of related events. """ __slots__ = ("__duration_micros", "__reply") def __init__(self, duration, reply, command_name, request_id, connection_id, operation_id): super(CommandSucceededEvent, self).__init__( command_name, request_id, connection_id, operation_id) self.__duration_micros = _to_micros(duration) if command_name.lower() in _SENSITIVE_COMMANDS: self.__reply = {} else: self.__reply = reply @property def duration_micros(self): """The duration of this operation in microseconds.""" return self.__duration_micros @property def reply(self): """The server failure document for this operation.""" return self.__reply class CommandFailedEvent(_CommandEvent): """Event published when a command fails. :Parameters: - `duration`: The command duration as a datetime.timedelta. - `failure`: The server reply document. - `command_name`: The command name. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `operation_id`: An optional identifier for a series of related events. """ __slots__ = ("__duration_micros", "__failure") def __init__(self, duration, failure, *args): super(CommandFailedEvent, self).__init__(*args) self.__duration_micros = _to_micros(duration) self.__failure = failure @property def duration_micros(self): """The duration of this operation in microseconds.""" return self.__duration_micros @property def failure(self): """The server failure document for this operation.""" return self.__failure class _ServerEvent(object): """Base class for server events.""" __slots__ = ("__server_address", "__topology_id") def __init__(self, server_address, topology_id): self.__server_address = server_address self.__topology_id = topology_id @property def server_address(self): """The address (host/port pair) of the server""" return self.__server_address @property def topology_id(self): """A unique identifier for the topology this server is a part of.""" return self.__topology_id class ServerDescriptionChangedEvent(_ServerEvent): """Published when server description changes. .. versionadded:: 3.3 """ __slots__ = ('__previous_description', '__new_description') def __init__(self, previous_description, new_description, *args): super(ServerDescriptionChangedEvent, self).__init__(*args) self.__previous_description = previous_description self.__new_description = new_description @property def previous_description(self): """The previous :class:`~pymongo.server_description.ServerDescription`.""" return self.__previous_description @property def new_description(self): """The new :class:`~pymongo.server_description.ServerDescription`.""" return self.__new_description class ServerOpeningEvent(_ServerEvent): """Published when server is initialized. .. versionadded:: 3.3 """ __slots__ = () class ServerClosedEvent(_ServerEvent): """Published when server is closed. .. versionadded:: 3.3 """ __slots__ = () class TopologyEvent(object): """Base class for topology description events.""" __slots__ = ('__topology_id') def __init__(self, topology_id): self.__topology_id = topology_id @property def topology_id(self): """A unique identifier for the topology this server is a part of.""" return self.__topology_id class TopologyDescriptionChangedEvent(TopologyEvent): """Published when the topology description changes. .. versionadded:: 3.3 """ __slots__ = ('__previous_description', '__new_description') def __init__(self, previous_description, new_description, *args): super(TopologyDescriptionChangedEvent, self).__init__(*args) self.__previous_description = previous_description self.__new_description = new_description @property def previous_description(self): """The previous :class:`~pymongo.topology_description.TopologyDescription`.""" return self.__previous_description @property def new_description(self): """The new :class:`~pymongo.topology_description.TopologyDescription`.""" return self.__new_description class TopologyOpenedEvent(TopologyEvent): """Published when the topology is initialized. .. versionadded:: 3.3 """ __slots__ = () class TopologyClosedEvent(TopologyEvent): """Published when the topology is closed. .. versionadded:: 3.3 """ __slots__ = () class _ServerHeartbeatEvent(object): """Base class for server heartbeat events.""" __slots__ = ('__connection_id') def __init__(self, connection_id): self.__connection_id = connection_id @property def connection_id(self): """The address (host, port) of the server this heartbeat was sent to.""" return self.__connection_id class ServerHeartbeatStartedEvent(_ServerHeartbeatEvent): """Published when a heartbeat is started. .. versionadded:: 3.3 """ __slots__ = () class ServerHeartbeatSucceededEvent(_ServerHeartbeatEvent): """Fired when the server heartbeat succeeds. .. versionadded:: 3.3 """ __slots__ = ('__duration', '__reply') def __init__(self, duration, reply, *args): super(ServerHeartbeatSucceededEvent, self).__init__(*args) self.__duration = duration self.__reply = reply @property def duration(self): """The duration of this heartbeat in microseconds.""" return self.__duration @property def reply(self): """An instance of :class:`~pymongo.ismaster.IsMaster`.""" return self.__reply class ServerHeartbeatFailedEvent(_ServerHeartbeatEvent): """Fired when the server heartbeat fails, either with an "ok: 0" or a socket exception. .. versionadded:: 3.3 """ __slots__ = ('__duration', '__reply') def __init__(self, duration, reply, *args): super(ServerHeartbeatFailedEvent, self).__init__(*args) self.__duration = duration self.__reply = reply @property def duration(self): """The duration of this heartbeat in microseconds.""" return self.__duration @property def reply(self): """A subclass of :exc:`Exception`.""" return self.__reply class _EventListeners(object): """Configure event listeners for a client instance. Any event listeners registered globally are included by default. :Parameters: - `listeners`: A list of event listeners. """ def __init__(self, listeners): self.__command_listeners = _LISTENERS.command_listeners[:] self.__server_listeners = _LISTENERS.server_listeners[:] lst = _LISTENERS.server_heartbeat_listeners self.__server_heartbeat_listeners = lst[:] self.__topology_listeners = _LISTENERS.topology_listeners[:] if listeners is not None: for lst in listeners: if isinstance(lst, CommandListener): self.__command_listeners.append(lst) if isinstance(lst, ServerListener): self.__server_listeners.append(lst) if isinstance(lst, ServerHeartbeatListener): self.__server_heartbeat_listeners.append(lst) if isinstance(lst, TopologyListener): self.__topology_listeners.append(lst) self.__enabled_for_commands = bool(self.__command_listeners) self.__enabled_for_server = bool(self.__server_listeners) self.__enabled_for_server_heartbeat = bool( self.__server_heartbeat_listeners) self.__enabled_for_topology = bool(self.__topology_listeners) @property def enabled_for_commands(self): """Are any CommandListener instances registered?""" return self.__enabled_for_commands @property def enabled_for_server(self): """Are any ServerListener instances registered?""" return self.__enabled_for_server @property def enabled_for_server_heartbeat(self): """Are any ServerHeartbeatListener instances registered?""" return self.__enabled_for_server_heartbeat @property def enabled_for_topology(self): """Are any TopologyListener instances registered?""" return self.__enabled_for_topology def event_listeners(self): """List of registered event listeners.""" return (self.__command_listeners[:], self.__server_heartbeat_listeners[:], self.__server_listeners[:], self.__topology_listeners[:]) def publish_command_start(self, command, database_name, request_id, connection_id, op_id=None): """Publish a CommandStartedEvent to all command listeners. :Parameters: - `command`: The command document. - `database_name`: The name of the database this command was run against. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `op_id`: The (optional) operation id for this operation. """ if op_id is None: op_id = request_id event = CommandStartedEvent( command, database_name, request_id, connection_id, op_id) for subscriber in self.__command_listeners: try: subscriber.started(event) except Exception: _handle_exception() def publish_command_success(self, duration, reply, command_name, request_id, connection_id, op_id=None): """Publish a CommandSucceededEvent to all command listeners. :Parameters: - `duration`: The command duration as a datetime.timedelta. - `reply`: The server reply document. - `command_name`: The command name. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `op_id`: The (optional) operation id for this operation. """ if op_id is None: op_id = request_id event = CommandSucceededEvent( duration, reply, command_name, request_id, connection_id, op_id) for subscriber in self.__command_listeners: try: subscriber.succeeded(event) except Exception: _handle_exception() def publish_command_failure(self, duration, failure, command_name, request_id, connection_id, op_id=None): """Publish a CommandFailedEvent to all command listeners. :Parameters: - `duration`: The command duration as a datetime.timedelta. - `failure`: The server reply document or failure description document. - `command_name`: The command name. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `op_id`: The (optional) operation id for this operation. """ if op_id is None: op_id = request_id event = CommandFailedEvent( duration, failure, command_name, request_id, connection_id, op_id) for subscriber in self.__command_listeners: try: subscriber.failed(event) except Exception: _handle_exception() def publish_server_heartbeat_started(self, connection_id): """Publish a ServerHeartbeatStartedEvent to all server heartbeat listeners. :Parameters: - `connection_id`: The address (host/port pair) of the connection. """ event = ServerHeartbeatStartedEvent(connection_id) for subscriber in self.__server_heartbeat_listeners: try: subscriber.started(event) except Exception: _handle_exception() def publish_server_heartbeat_succeeded(self, connection_id, duration, reply): """Publish a ServerHeartbeatSucceededEvent to all server heartbeat listeners. :Parameters: - `connection_id`: The address (host/port pair) of the connection. - `duration`: The execution time of the event in the highest possible resolution for the platform. - `reply`: The command reply. """ event = ServerHeartbeatSucceededEvent(duration, reply, connection_id) for subscriber in self.__server_heartbeat_listeners: try: subscriber.succeeded(event) except Exception: _handle_exception() def publish_server_heartbeat_failed(self, connection_id, duration, reply): """Publish a ServerHeartbeatFailedEvent to all server heartbeat listeners. :Parameters: - `connection_id`: The address (host/port pair) of the connection. - `duration`: The execution time of the event in the highest possible resolution for the platform. - `reply`: The command reply. """ event = ServerHeartbeatFailedEvent(duration, reply, connection_id) for subscriber in self.__server_heartbeat_listeners: try: subscriber.failed(event) except Exception: _handle_exception() def publish_server_opened(self, server_address, topology_id): """Publish a ServerOpeningEvent to all server listeners. :Parameters: - `server_address`: The address (host/port pair) of the server. - `topology_id`: A unique identifier for the topology this server is a part of. """ event = ServerOpeningEvent(server_address, topology_id) for subscriber in self.__server_listeners: try: subscriber.opened(event) except Exception: _handle_exception() def publish_server_closed(self, server_address, topology_id): """Publish a ServerClosedEvent to all server listeners. :Parameters: - `server_address`: The address (host/port pair) of the server. - `topology_id`: A unique identifier for the topology this server is a part of. """ event = ServerClosedEvent(server_address, topology_id) for subscriber in self.__server_listeners: try: subscriber.closed(event) except Exception: _handle_exception() def publish_server_description_changed(self, previous_description, new_description, server_address, topology_id): """Publish a ServerDescriptionChangedEvent to all server listeners. :Parameters: - `previous_description`: The previous server description. - `server_address`: The address (host/port pair) of the server. - `new_description`: The new server description. - `topology_id`: A unique identifier for the topology this server is a part of. """ event = ServerDescriptionChangedEvent(previous_description, new_description, server_address, topology_id) for subscriber in self.__server_listeners: try: subscriber.description_changed(event) except Exception: _handle_exception() def publish_topology_opened(self, topology_id): """Publish a TopologyOpenedEvent to all topology listeners. :Parameters: - `topology_id`: A unique identifier for the topology this server is a part of. """ event = TopologyOpenedEvent(topology_id) for subscriber in self.__topology_listeners: try: subscriber.opened(event) except Exception: _handle_exception() def publish_topology_closed(self, topology_id): """Publish a TopologyClosedEvent to all topology listeners. :Parameters: - `topology_id`: A unique identifier for the topology this server is a part of. """ event = TopologyClosedEvent(topology_id) for subscriber in self.__topology_listeners: try: subscriber.closed(event) except Exception: _handle_exception() def publish_topology_description_changed(self, previous_description, new_description, topology_id): """Publish a TopologyDescriptionChangedEvent to all topology listeners. :Parameters: - `previous_description`: The previous topology description. - `new_description`: The new topology description. - `topology_id`: A unique identifier for the topology this server is a part of. """ event = TopologyDescriptionChangedEvent(previous_description, new_description, topology_id) for subscriber in self.__topology_listeners: try: subscriber.description_changed(event) except Exception: _handle_exception() pymongo-3.6.1/pymongo/cursor_manager.py0000644000076600000240000000405013245621354020510 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """DEPRECATED - A manager to handle when cursors are killed after they are closed. New cursor managers should be defined as subclasses of CursorManager and can be installed on a client by calling :meth:`~pymongo.mongo_client.MongoClient.set_cursor_manager`. .. versionchanged:: 3.3 Deprecated, for real this time. .. versionchanged:: 3.0 Undeprecated. :meth:`~pymongo.cursor_manager.CursorManager.close` now requires an `address` argument. The ``BatchCursorManager`` class is removed. """ import warnings import weakref from bson.py3compat import integer_types class CursorManager(object): """DEPRECATED - The cursor manager base class.""" def __init__(self, client): """Instantiate the manager. :Parameters: - `client`: a MongoClient """ warnings.warn( "Cursor managers are deprecated.", DeprecationWarning, stacklevel=2) self.__client = weakref.ref(client) def close(self, cursor_id, address): """Kill a cursor. Raises TypeError if cursor_id is not an instance of (int, long). :Parameters: - `cursor_id`: cursor id to close - `address`: the cursor's server's (host, port) pair .. versionchanged:: 3.0 Now requires an `address` argument. """ if not isinstance(cursor_id, integer_types): raise TypeError("cursor_id must be an integer") self.__client().kill_cursors([cursor_id], address) pymongo-3.6.1/pymongo/database.py0000644000076600000240000014523713245621354017262 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Database level operations.""" import warnings from bson.code import Code from bson.codec_options import DEFAULT_CODEC_OPTIONS from bson.dbref import DBRef from bson.py3compat import iteritems, string_type, _unicode from bson.son import SON from pymongo import auth, common from pymongo.collection import Collection from pymongo.command_cursor import CommandCursor from pymongo.errors import (CollectionInvalid, ConfigurationError, InvalidName, OperationFailure) from pymongo.message import _first_batch from pymongo.read_preferences import ReadPreference from pymongo.son_manipulator import SONManipulator from pymongo.write_concern import WriteConcern _INDEX_REGEX = {"name": {"$regex": "^(?!.*\$)"}} _SYSTEM_FILTER = {"filter": {"name": {"$regex": "^(?!system\.)"}}} def _check_name(name): """Check if a database name is valid. """ if not name: raise InvalidName("database name cannot be the empty string") for invalid_char in [' ', '.', '$', '/', '\\', '\x00', '"']: if invalid_char in name: raise InvalidName("database names cannot contain the " "character %r" % invalid_char) class Database(common.BaseObject): """A Mongo database. """ def __init__(self, client, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None): """Get a database by client and name. Raises :class:`TypeError` if `name` is not an instance of :class:`basestring` (:class:`str` in python 3). Raises :class:`~pymongo.errors.InvalidName` if `name` is not a valid database name. :Parameters: - `client`: A :class:`~pymongo.mongo_client.MongoClient` instance. - `name`: The database name. - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) client.codec_options is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) client.read_preference is used. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) client.write_concern is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) client.read_concern is used. .. mongodoc:: databases .. versionchanged:: 3.2 Added the read_concern option. .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. :class:`~pymongo.database.Database` no longer returns an instance of :class:`~pymongo.collection.Collection` for attribute names with leading underscores. You must use dict-style lookups instead:: db['__my_collection__'] Not: db.__my_collection__ """ super(Database, self).__init__( codec_options or client.codec_options, read_preference or client.read_preference, write_concern or client.write_concern, read_concern or client.read_concern) if not isinstance(name, string_type): raise TypeError("name must be an instance " "of %s" % (string_type.__name__,)) if name != '$external': _check_name(name) self.__name = _unicode(name) self.__client = client self.__incoming_manipulators = [] self.__incoming_copying_manipulators = [] self.__outgoing_manipulators = [] self.__outgoing_copying_manipulators = [] def add_son_manipulator(self, manipulator): """Add a new son manipulator to this database. **DEPRECATED** - `add_son_manipulator` is deprecated. .. versionchanged:: 3.0 Deprecated add_son_manipulator. """ warnings.warn("add_son_manipulator is deprecated", DeprecationWarning, stacklevel=2) base = SONManipulator() def method_overwritten(instance, method): """Test if this method has been overridden.""" return (getattr( instance, method).__func__ != getattr(base, method).__func__) if manipulator.will_copy(): if method_overwritten(manipulator, "transform_incoming"): self.__incoming_copying_manipulators.insert(0, manipulator) if method_overwritten(manipulator, "transform_outgoing"): self.__outgoing_copying_manipulators.insert(0, manipulator) else: if method_overwritten(manipulator, "transform_incoming"): self.__incoming_manipulators.insert(0, manipulator) if method_overwritten(manipulator, "transform_outgoing"): self.__outgoing_manipulators.insert(0, manipulator) @property def system_js(self): """**DEPRECATED**: :class:`SystemJS` helper for this :class:`Database`. See the documentation for :class:`SystemJS` for more details. """ return SystemJS(self) @property def client(self): """The client instance for this :class:`Database`.""" return self.__client @property def name(self): """The name of this :class:`Database`.""" return self.__name @property def incoming_manipulators(self): """**DEPRECATED**: All incoming SON manipulators. .. versionchanged:: 3.5 Deprecated. .. versionadded:: 2.0 """ warnings.warn("Database.incoming_manipulators() is deprecated", DeprecationWarning, stacklevel=2) return [manipulator.__class__.__name__ for manipulator in self.__incoming_manipulators] @property def incoming_copying_manipulators(self): """**DEPRECATED**: All incoming SON copying manipulators. .. versionchanged:: 3.5 Deprecated. .. versionadded:: 2.0 """ warnings.warn("Database.incoming_copying_manipulators() is deprecated", DeprecationWarning, stacklevel=2) return [manipulator.__class__.__name__ for manipulator in self.__incoming_copying_manipulators] @property def outgoing_manipulators(self): """**DEPRECATED**: All outgoing SON manipulators. .. versionchanged:: 3.5 Deprecated. .. versionadded:: 2.0 """ warnings.warn("Database.outgoing_manipulators() is deprecated", DeprecationWarning, stacklevel=2) return [manipulator.__class__.__name__ for manipulator in self.__outgoing_manipulators] @property def outgoing_copying_manipulators(self): """**DEPRECATED**: All outgoing SON copying manipulators. .. versionchanged:: 3.5 Deprecated. .. versionadded:: 2.0 """ warnings.warn("Database.outgoing_copying_manipulators() is deprecated", DeprecationWarning, stacklevel=2) return [manipulator.__class__.__name__ for manipulator in self.__outgoing_copying_manipulators] def __eq__(self, other): if isinstance(other, Database): return (self.__client == other.client and self.__name == other.name) return NotImplemented def __ne__(self, other): return not self == other def __repr__(self): return "Database(%r, %r)" % (self.__client, self.__name) def __getattr__(self, name): """Get a collection of this database by name. Raises InvalidName if an invalid collection name is used. :Parameters: - `name`: the name of the collection to get """ if name.startswith('_'): raise AttributeError( "Database has no attribute %r. To access the %s" " collection, use database[%r]." % (name, name, name)) return self.__getitem__(name) def __getitem__(self, name): """Get a collection of this database by name. Raises InvalidName if an invalid collection name is used. :Parameters: - `name`: the name of the collection to get """ return Collection(self, name) def get_collection(self, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None): """Get a :class:`~pymongo.collection.Collection` with the given name and options. Useful for creating a :class:`~pymongo.collection.Collection` with different codec options, read preference, and/or write concern from this :class:`Database`. >>> db.read_preference Primary() >>> coll1 = db.test >>> coll1.read_preference Primary() >>> from pymongo import ReadPreference >>> coll2 = db.get_collection( ... 'test', read_preference=ReadPreference.SECONDARY) >>> coll2.read_preference Secondary(tag_sets=None) :Parameters: - `name`: The name of the collection - a string. - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`Database` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`Database` is used. See :mod:`~pymongo.read_preferences` for options. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`Database` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`Database` is used. """ return Collection( self, name, False, codec_options, read_preference, write_concern, read_concern) def _collection_default_options(self, name, **kargs): """Get a Collection instance with the default settings.""" wc = (self.write_concern if self.write_concern.acknowledged else WriteConcern()) return self.get_collection( name, codec_options=DEFAULT_CODEC_OPTIONS, read_preference=ReadPreference.PRIMARY, write_concern=wc) def create_collection(self, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None, session=None, **kwargs): """Create a new :class:`~pymongo.collection.Collection` in this database. Normally collection creation is automatic. This method should only be used to specify options on creation. :class:`~pymongo.errors.CollectionInvalid` will be raised if the collection already exists. Options should be passed as keyword arguments to this method. Supported options vary with MongoDB release. Some examples include: - "size": desired initial size for the collection (in bytes). For capped collections this size is the max size of the collection. - "capped": if True, this is a capped collection - "max": maximum number of objects if capped (optional) See the MongoDB documentation for a full list of supported options by server version. :Parameters: - `name`: the name of the collection to create - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`Database` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`Database` is used. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`Database` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`Database` is used. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional keyword arguments will be passed as options for the create collection command .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Added the collation option. .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. .. versionchanged:: 2.2 Removed deprecated argument: options """ with self.__client._tmp_session(session) as s: if name in self.collection_names(session=s): raise CollectionInvalid("collection %s already exists" % name) return Collection(self, name, True, codec_options, read_preference, write_concern, read_concern, session=s, **kwargs) def _apply_incoming_manipulators(self, son, collection): """Apply incoming manipulators to `son`.""" for manipulator in self.__incoming_manipulators: son = manipulator.transform_incoming(son, collection) return son def _apply_incoming_copying_manipulators(self, son, collection): """Apply incoming copying manipulators to `son`.""" for manipulator in self.__incoming_copying_manipulators: son = manipulator.transform_incoming(son, collection) return son def _fix_incoming(self, son, collection): """Apply manipulators to an incoming SON object before it gets stored. :Parameters: - `son`: the son object going into the database - `collection`: the collection the son object is being saved in """ son = self._apply_incoming_manipulators(son, collection) son = self._apply_incoming_copying_manipulators(son, collection) return son def _fix_outgoing(self, son, collection): """Apply manipulators to a SON object as it comes out of the database. :Parameters: - `son`: the son object coming out of the database - `collection`: the collection the son object was saved in """ for manipulator in reversed(self.__outgoing_manipulators): son = manipulator.transform_outgoing(son, collection) for manipulator in reversed(self.__outgoing_copying_manipulators): son = manipulator.transform_outgoing(son, collection) return son def _command(self, sock_info, command, slave_ok=False, value=1, check=True, allowable_errors=None, read_preference=ReadPreference.PRIMARY, codec_options=DEFAULT_CODEC_OPTIONS, write_concern=None, parse_write_concern_error=False, session=None, **kwargs): """Internal command helper.""" if isinstance(command, string_type): command = SON([(command, value)]) if sock_info.max_wire_version >= 5 and write_concern: command['writeConcern'] = write_concern.document command.update(kwargs) with self.__client._tmp_session(session) as s: return sock_info.command( self.__name, command, slave_ok, read_preference, codec_options, check, allowable_errors, parse_write_concern_error=parse_write_concern_error, session=s, client=self.__client) def command(self, command, value=1, check=True, allowable_errors=None, read_preference=ReadPreference.PRIMARY, codec_options=DEFAULT_CODEC_OPTIONS, session=None, **kwargs): """Issue a MongoDB command. Send command `command` to the database and return the response. If `command` is an instance of :class:`basestring` (:class:`str` in python 3) then the command {`command`: `value`} will be sent. Otherwise, `command` must be an instance of :class:`dict` and will be sent as is. Any additional keyword arguments will be added to the final command document before it is sent. For example, a command like ``{buildinfo: 1}`` can be sent using: >>> db.command("buildinfo") For a command where the value matters, like ``{collstats: collection_name}`` we can do: >>> db.command("collstats", collection_name) For commands that take additional arguments we can use kwargs. So ``{filemd5: object_id, root: file_root}`` becomes: >>> db.command("filemd5", object_id, root=file_root) :Parameters: - `command`: document representing the command to be issued, or the name of the command (for simple commands only). .. note:: the order of keys in the `command` document is significant (the "verb" must come first), so commands which require multiple keys (e.g. `findandmodify`) should use an instance of :class:`~bson.son.SON` or a string and kwargs instead of a Python `dict`. - `value` (optional): value to use for the command verb when `command` is passed as a string - `check` (optional): check the response for errors, raising :class:`~pymongo.errors.OperationFailure` if there are any - `allowable_errors`: if `check` is ``True``, error messages in this list will be ignored by error-checking - `read_preference`: The read preference for this operation. See :mod:`~pymongo.read_preferences` for options. - `codec_options`: A :class:`~bson.codec_options.CodecOptions` instance. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional keyword arguments will be added to the command document before it is sent .. note:: :meth:`command` does **not** obey :attr:`read_preference` or :attr:`codec_options`. You must use the `read_preference` and `codec_options` parameters instead. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.0 Removed the `as_class`, `fields`, `uuid_subtype`, `tag_sets`, and `secondary_acceptable_latency_ms` option. Removed `compile_re` option: PyMongo now always represents BSON regular expressions as :class:`~bson.regex.Regex` objects. Use :meth:`~bson.regex.Regex.try_compile` to attempt to convert from a BSON regular expression to a Python regular expression object. Added the `codec_options` parameter. .. versionchanged:: 2.7 Added `compile_re` option. If set to False, PyMongo represented BSON regular expressions as :class:`~bson.regex.Regex` objects instead of attempting to compile BSON regular expressions as Python native regular expressions, thus preventing errors for some incompatible patterns, see `PYTHON-500`_. .. versionchanged:: 2.3 Added `tag_sets` and `secondary_acceptable_latency_ms` options. .. versionchanged:: 2.2 Added support for `as_class` - the class you want to use for the resulting documents .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500 .. mongodoc:: commands """ client = self.__client with client._socket_for_reads(read_preference) as (sock_info, slave_ok): return self._command(sock_info, command, slave_ok, value, check, allowable_errors, read_preference, codec_options, session=session, **kwargs) def _list_collections(self, sock_info, slave_okay, session=None, **kwargs): """Internal listCollections helper.""" coll = self["$cmd"] if sock_info.max_wire_version > 2: cmd = SON([("listCollections", 1), ("cursor", {})]) cmd.update(kwargs) with self.__client._tmp_session( session, close=False) as tmp_session: cursor = self._command( sock_info, cmd, slave_okay, session=tmp_session)["cursor"] return CommandCursor( coll, cursor, sock_info.address, session=tmp_session, explicit_session=session is not None) else: match = _INDEX_REGEX if "filter" in kwargs: match = {"$and": [_INDEX_REGEX, kwargs["filter"]]} dblen = len(self.name.encode("utf8") + b".") pipeline = [ {"$project": {"name": {"$substr": ["$name", dblen, -1]}, "options": 1}}, {"$match": match} ] cmd = SON([("aggregate", "system.namespaces"), ("pipeline", pipeline), ("cursor", kwargs.get("cursor", {}))]) cursor = self._command(sock_info, cmd, slave_okay)["cursor"] return CommandCursor(coll, cursor, sock_info.address) def list_collections(self, session=None, **kwargs): """Get a cursor over the collectons of this database. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): Optional parameters of the `listCollections command `_ can be passed as keyword arguments to this method. The supported options differ by server version. :Returns: An instance of :class:`~pymongo.command_cursor.CommandCursor`. .. versionadded:: 3.6 """ with self.__client._socket_for_reads( ReadPreference.PRIMARY) as (sock_info, slave_okay): return self._list_collections( sock_info, slave_okay, session=session, **kwargs) def list_collection_names(self, session=None): """Get a list of all the collection names in this database. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionadded:: 3.6 """ return [result["name"] for result in self.list_collections(session=session)] def collection_names(self, include_system_collections=True, session=None): """Get a list of all the collection names in this database. :Parameters: - `include_system_collections` (optional): if ``False`` list will not include system collections (e.g ``system.indexes``) - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. """ kws = {} if include_system_collections else _SYSTEM_FILTER return [result["name"] for result in self.list_collections(session=session, **kws)] def drop_collection(self, name_or_collection, session=None): """Drop a collection. :Parameters: - `name_or_collection`: the name of a collection to drop or the collection object itself - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. note:: The :attr:`~pymongo.database.Database.write_concern` of this database is automatically applied to this operation when using MongoDB >= 3.4. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Apply this database's write concern automatically to this operation when connected to MongoDB >= 3.4. """ name = name_or_collection if isinstance(name, Collection): name = name.name if not isinstance(name, string_type): raise TypeError("name_or_collection must be an " "instance of %s" % (string_type.__name__,)) self.__client._purge_index(self.__name, name) with self.__client._socket_for_reads( ReadPreference.PRIMARY) as (sock_info, slave_ok): return self._command( sock_info, 'drop', slave_ok, _unicode(name), allowable_errors=['ns not found'], write_concern=self.write_concern, parse_write_concern_error=True, session=session) def validate_collection(self, name_or_collection, scandata=False, full=False, session=None): """Validate a collection. Returns a dict of validation info. Raises CollectionInvalid if validation fails. :Parameters: - `name_or_collection`: A Collection object or the name of a collection to validate. - `scandata`: Do extra checks beyond checking the overall structure of the collection. - `full`: Have the server do a more thorough scan of the collection. Use with `scandata` for a thorough scan of the structure of the collection and the individual documents. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. """ name = name_or_collection if isinstance(name, Collection): name = name.name if not isinstance(name, string_type): raise TypeError("name_or_collection must be an instance of " "%s or Collection" % (string_type.__name__,)) result = self.command("validate", _unicode(name), scandata=scandata, full=full, session=session) valid = True # Pre 1.9 results if "result" in result: info = result["result"] if info.find("exception") != -1 or info.find("corrupt") != -1: raise CollectionInvalid("%s invalid: %s" % (name, info)) # Sharded results elif "raw" in result: for _, res in iteritems(result["raw"]): if "result" in res: info = res["result"] if (info.find("exception") != -1 or info.find("corrupt") != -1): raise CollectionInvalid("%s invalid: " "%s" % (name, info)) elif not res.get("valid", False): valid = False break # Post 1.9 non-sharded results. elif not result.get("valid", False): valid = False if not valid: raise CollectionInvalid("%s invalid: %r" % (name, result)) return result def current_op(self, include_all=False, session=None): """Get information on operations currently running. :Parameters: - `include_all` (optional): if ``True`` also list currently idle operations in the result - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. """ cmd = SON([("currentOp", 1), ("$all", include_all)]) with self.__client._socket_for_writes() as sock_info: if sock_info.max_wire_version >= 4: with self.__client._tmp_session(session) as s: return sock_info.command("admin", cmd, session=s, client=self.__client) else: spec = {"$all": True} if include_all else {} return _first_batch(sock_info, "admin", "$cmd.sys.inprog", spec, -1, True, self.codec_options, ReadPreference.PRIMARY, cmd, self.client._event_listeners) def profiling_level(self, session=None): """Get the database's current profiling level. Returns one of (:data:`~pymongo.OFF`, :data:`~pymongo.SLOW_ONLY`, :data:`~pymongo.ALL`). :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. .. mongodoc:: profiling """ result = self.command("profile", -1, session=session) assert result["was"] >= 0 and result["was"] <= 2 return result["was"] def set_profiling_level(self, level, slow_ms=None, session=None): """Set the database's profiling level. :Parameters: - `level`: Specifies a profiling level, see list of possible values below. - `slow_ms`: Optionally modify the threshold for the profile to consider a query or operation. Even if the profiler is off queries slower than the `slow_ms` level will get written to the logs. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. Possible `level` values: +----------------------------+------------------------------------+ | Level | Setting | +============================+====================================+ | :data:`~pymongo.OFF` | Off. No profiling. | +----------------------------+------------------------------------+ | :data:`~pymongo.SLOW_ONLY` | On. Only includes slow operations. | +----------------------------+------------------------------------+ | :data:`~pymongo.ALL` | On. Includes all operations. | +----------------------------+------------------------------------+ Raises :class:`ValueError` if level is not one of (:data:`~pymongo.OFF`, :data:`~pymongo.SLOW_ONLY`, :data:`~pymongo.ALL`). .. versionchanged:: 3.6 Added ``session`` parameter. .. mongodoc:: profiling """ if not isinstance(level, int) or level < 0 or level > 2: raise ValueError("level must be one of (OFF, SLOW_ONLY, ALL)") if slow_ms is not None and not isinstance(slow_ms, int): raise TypeError("slow_ms must be an integer") if slow_ms is not None: self.command("profile", level, slowms=slow_ms, session=session) else: self.command("profile", level, session=session) def profiling_info(self, session=None): """Returns a list containing current profiling information. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. .. mongodoc:: profiling """ return list(self["system.profile"].find(session=session)) def error(self): """**DEPRECATED**: Get the error if one occurred on the last operation. This method is obsolete: all MongoDB write operations (insert, update, remove, and so on) use the write concern ``w=1`` and report their errors by default. .. versionchanged:: 2.8 Deprecated. """ warnings.warn("Database.error() is deprecated", DeprecationWarning, stacklevel=2) error = self.command("getlasterror") error_msg = error.get("err", "") if error_msg is None: return None if error_msg.startswith("not master"): # Reset primary server and request check, if another thread isn't # doing so already. primary = self.__client.primary if primary: self.__client._reset_server_and_request_check(primary) return error def last_status(self): """**DEPRECATED**: Get status information from the last operation. This method is obsolete: all MongoDB write operations (insert, update, remove, and so on) use the write concern ``w=1`` and report their errors by default. Returns a SON object with status information. .. versionchanged:: 2.8 Deprecated. """ warnings.warn("last_status() is deprecated", DeprecationWarning, stacklevel=2) return self.command("getlasterror") def previous_error(self): """**DEPRECATED**: Get the most recent error on this database. This method is obsolete: all MongoDB write operations (insert, update, remove, and so on) use the write concern ``w=1`` and report their errors by default. Only returns errors that have occurred since the last call to :meth:`reset_error_history`. Returns None if no such errors have occurred. .. versionchanged:: 2.8 Deprecated. """ warnings.warn("previous_error() is deprecated", DeprecationWarning, stacklevel=2) error = self.command("getpreverror") if error.get("err", 0) is None: return None return error def reset_error_history(self): """**DEPRECATED**: Reset the error history of this database. This method is obsolete: all MongoDB write operations (insert, update, remove, and so on) use the write concern ``w=1`` and report their errors by default. Calls to :meth:`previous_error` will only return errors that have occurred since the most recent call to this method. .. versionchanged:: 2.8 Deprecated. """ warnings.warn("reset_error_history() is deprecated", DeprecationWarning, stacklevel=2) self.command("reseterror") def __iter__(self): return self def __next__(self): raise TypeError("'Database' object is not iterable") next = __next__ def _default_role(self, read_only): """Return the default user role for this database.""" if self.name == "admin": if read_only: return "readAnyDatabase" else: return "root" else: if read_only: return "read" else: return "dbOwner" def _create_or_update_user( self, create, name, password, read_only, session=None, **kwargs): """Use a command to create (if create=True) or modify a user. """ opts = {} if read_only or (create and "roles" not in kwargs): warnings.warn("Creating a user with the read_only option " "or without roles is deprecated in MongoDB " ">= 2.6", DeprecationWarning) opts["roles"] = [self._default_role(read_only)] elif read_only: warnings.warn("The read_only option is deprecated in MongoDB " ">= 2.6, use 'roles' instead", DeprecationWarning) if password is not None: # We always salt and hash client side. if "digestPassword" in kwargs: raise ConfigurationError("The digestPassword option is not " "supported via add_user. Please use " "db.command('createUser', ...) " "instead for this option.") opts["pwd"] = auth._password_digest(name, password) opts["digestPassword"] = False # Don't send {} as writeConcern. if self.write_concern.acknowledged and self.write_concern.document: opts["writeConcern"] = self.write_concern.document opts.update(kwargs) if create: command_name = "createUser" else: command_name = "updateUser" self.command(command_name, name, session=session, **opts) def add_user(self, name, password=None, read_only=None, session=None, **kwargs): """**DEPRECATED**: Create user `name` with password `password`. Add a new user with permissions for this :class:`Database`. .. note:: Will change the password if user `name` already exists. .. note:: add_user is deprecated and will be removed in PyMongo 4.0. Starting with MongoDB 2.6 user management is handled with four database commands, createUser_, usersInfo_, updateUser_, and dropUser_. To create a user:: db.command("createUser", "admin", pwd="password", roles=["root"]) To create a read-only user:: db.command("createUser", "user", pwd="password", roles=["read"]) To change a password:: db.command("updateUser", "user", pwd="newpassword") Or change roles:: db.command("updateUser", "user", roles=["readWrite"]) .. _createUser: https://docs.mongodb.com/manual/reference/command/createUser/ .. _usersInfo: https://docs.mongodb.com/manual/reference/command/usersInfo/ .. _updateUser: https://docs.mongodb.com/manual/reference/command/updateUser/ .. _dropUser: https://docs.mongodb.com/manual/reference/command/createUser/ :Parameters: - `name`: the name of the user to create - `password` (optional): the password of the user to create. Can not be used with the ``userSource`` argument. - `read_only` (optional): if ``True`` the user will be read only - `**kwargs` (optional): optional fields for the user document (e.g. ``userSource``, ``otherDBRoles``, or ``roles``). See ``_ for more information. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. Deprecated add_user. .. versionchanged:: 2.5 Added kwargs support for optional fields introduced in MongoDB 2.4 .. versionchanged:: 2.2 Added support for read only users """ warnings.warn("add_user is deprecated and will be removed in PyMongo " "4.0. Use db.command with createUser or updateUser " "instead", DeprecationWarning, stacklevel=2) if not isinstance(name, string_type): raise TypeError("name must be an " "instance of %s" % (string_type.__name__,)) if password is not None: if not isinstance(password, string_type): raise TypeError("password must be an " "instance of %s" % (string_type.__name__,)) if len(password) == 0: raise ValueError("password can't be empty") if read_only is not None: read_only = common.validate_boolean('read_only', read_only) if 'roles' in kwargs: raise ConfigurationError("Can not use " "read_only and roles together") try: uinfo = self.command("usersInfo", name, session=session) # Create the user if not found in uinfo, otherwise update one. self._create_or_update_user( (not uinfo["users"]), name, password, read_only, session=session, **kwargs) except OperationFailure as exc: # Unauthorized. Attempt to create the user in case of # localhost exception. if exc.code == 13: self._create_or_update_user( True, name, password, read_only, session=session, **kwargs) else: raise def remove_user(self, name, session=None): """**DEPRECATED**: Remove user `name` from this :class:`Database`. User `name` will no longer have permissions to access this :class:`Database`. .. note:: remove_user is deprecated and will be removed in PyMongo 4.0. Use the dropUser command instead:: db.command("dropUser", "user") :Parameters: - `name`: the name of the user to remove - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. Deprecated remove_user. """ warnings.warn("remove_user is deprecated and will be removed in " "PyMongo 4.0. Use db.command with dropUser " "instead", DeprecationWarning, stacklevel=2) cmd = SON([("dropUser", name)]) # Don't send {} as writeConcern. if self.write_concern.acknowledged and self.write_concern.document: cmd["writeConcern"] = self.write_concern.document self.command(cmd, session=session) def authenticate(self, name=None, password=None, source=None, mechanism='DEFAULT', **kwargs): """**DEPRECATED**: Authenticate to use this database. .. warning:: Starting in MongoDB 3.6, calling :meth:`authenticate` invalidates all existing cursors. It may also leave logical sessions open on the server for up to 30 minutes until they time out. Authentication lasts for the life of the underlying client instance, or until :meth:`logout` is called. Raises :class:`TypeError` if (required) `name`, (optional) `password`, or (optional) `source` is not an instance of :class:`basestring` (:class:`str` in python 3). .. note:: - This method authenticates the current connection, and will also cause all new :class:`~socket.socket` connections in the underlying client instance to be authenticated automatically. - Authenticating more than once on the same database with different credentials is not supported. You must call :meth:`logout` before authenticating with new credentials. - When sharing a client instance between multiple threads, all threads will share the authentication. If you need different authentication profiles for different purposes you must use distinct client instances. :Parameters: - `name`: the name of the user to authenticate. Optional when `mechanism` is MONGODB-X509 and the MongoDB server version is >= 3.4. - `password` (optional): the password of the user to authenticate. Not used with GSSAPI or MONGODB-X509 authentication. - `source` (optional): the database to authenticate on. If not specified the current database is used. - `mechanism` (optional): See :data:`~pymongo.auth.MECHANISMS` for options. By default, use SCRAM-SHA-1 with MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response protocol) for older servers. - `authMechanismProperties` (optional): Used to specify authentication mechanism specific options. To specify the service name for GSSAPI authentication pass authMechanismProperties='SERVICE_NAME:' .. versionchanged:: 3.5 Deprecated. Authenticating multiple users conflicts with support for logical sessions in MongoDB 3.6. To authenticate as multiple users, create multiple instances of MongoClient. .. versionadded:: 2.8 Use SCRAM-SHA-1 with MongoDB 3.0 and later. .. versionchanged:: 2.5 Added the `source` and `mechanism` parameters. :meth:`authenticate` now raises a subclass of :class:`~pymongo.errors.PyMongoError` if authentication fails due to invalid credentials or configuration issues. .. mongodoc:: authenticate """ if name is not None and not isinstance(name, string_type): raise TypeError("name must be an " "instance of %s" % (string_type.__name__,)) if password is not None and not isinstance(password, string_type): raise TypeError("password must be an " "instance of %s" % (string_type.__name__,)) if source is not None and not isinstance(source, string_type): raise TypeError("source must be an " "instance of %s" % (string_type.__name__,)) common.validate_auth_mechanism('mechanism', mechanism) validated_options = {} for option, value in iteritems(kwargs): normalized, val = common.validate_auth_option(option, value) validated_options[normalized] = val credentials = auth._build_credentials_tuple( mechanism, source or self.name, name, password, validated_options) self.client._cache_credentials( self.name, credentials, connect=True) return True def logout(self): """**DEPRECATED**: Deauthorize use of this database. .. warning:: Starting in MongoDB 3.6, calling :meth:`logout` invalidates all existing cursors. It may also leave logical sessions open on the server for up to 30 minutes until they time out. """ warnings.warn("Database.logout() is deprecated", DeprecationWarning, stacklevel=2) # Sockets will be deauthenticated as they are used. self.client._purge_credentials(self.name) def dereference(self, dbref, session=None, **kwargs): """Dereference a :class:`~bson.dbref.DBRef`, getting the document it points to. Raises :class:`TypeError` if `dbref` is not an instance of :class:`~bson.dbref.DBRef`. Returns a document, or ``None`` if the reference does not point to a valid document. Raises :class:`ValueError` if `dbref` has a database specified that is different from the current database. :Parameters: - `dbref`: the reference - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`~pymongo.collection.Collection.find`. .. versionchanged:: 3.6 Added ``session`` parameter. """ if not isinstance(dbref, DBRef): raise TypeError("cannot dereference a %s" % type(dbref)) if dbref.database is not None and dbref.database != self.__name: raise ValueError("trying to dereference a DBRef that points to " "another database (%r not %r)" % (dbref.database, self.__name)) return self[dbref.collection].find_one( {"_id": dbref.id}, session=session, **kwargs) def eval(self, code, *args): """**DEPRECATED**: Evaluate a JavaScript expression in MongoDB. :Parameters: - `code`: string representation of JavaScript code to be evaluated - `args` (optional): additional positional arguments are passed to the `code` being evaluated .. warning:: the eval command is deprecated in MongoDB 3.0 and will be removed in a future server version. """ warnings.warn("Database.eval() is deprecated", DeprecationWarning, stacklevel=2) if not isinstance(code, Code): code = Code(code) result = self.command("$eval", code, args=args) return result.get("retval", None) def __call__(self, *args, **kwargs): """This is only here so that some API misusages are easier to debug. """ raise TypeError("'Database' object is not callable. If you meant to " "call the '%s' method on a '%s' object it is " "failing because no such method exists." % ( self.__name, self.__client.__class__.__name__)) class SystemJS(object): """**DEPRECATED**: Helper class for dealing with stored JavaScript. """ def __init__(self, database): """**DEPRECATED**: Get a system js helper for the database `database`. SystemJS will be removed in PyMongo 4.0. """ warnings.warn("SystemJS is deprecated", DeprecationWarning, stacklevel=2) if not database.write_concern.acknowledged: database = database.client.get_database( database.name, write_concern=WriteConcern()) # can't just assign it since we've overridden __setattr__ object.__setattr__(self, "_db", database) def __setattr__(self, name, code): self._db.system.js.replace_one( {"_id": name}, {"_id": name, "value": Code(code)}, True) def __setitem__(self, name, code): self.__setattr__(name, code) def __delattr__(self, name): self._db.system.js.delete_one({"_id": name}) def __delitem__(self, name): self.__delattr__(name) def __getattr__(self, name): return lambda *args: self._db.eval(Code("function() { " "return this[name].apply(" "this, arguments); }", scope={'name': name}), *args) def __getitem__(self, name): return self.__getattr__(name) def list(self): """Get a list of the names of the functions stored in this database.""" return [x["_id"] for x in self._db.system.js.find(projection=["_id"])] pymongo-3.6.1/pymongo/ismaster.py0000644000076600000240000001053113245621354017331 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Parse a response to the 'ismaster' command.""" import itertools from bson.py3compat import imap from pymongo import common from pymongo.server_type import SERVER_TYPE def _get_server_type(doc): """Determine the server type from an ismaster response.""" if not doc.get('ok'): return SERVER_TYPE.Unknown if doc.get('isreplicaset'): return SERVER_TYPE.RSGhost elif doc.get('setName'): if doc.get('hidden'): return SERVER_TYPE.RSOther elif doc.get('ismaster'): return SERVER_TYPE.RSPrimary elif doc.get('secondary'): return SERVER_TYPE.RSSecondary elif doc.get('arbiterOnly'): return SERVER_TYPE.RSArbiter else: return SERVER_TYPE.RSOther elif doc.get('msg') == 'isdbgrid': return SERVER_TYPE.Mongos else: return SERVER_TYPE.Standalone class IsMaster(object): __slots__ = ('_doc', '_server_type', '_is_writable', '_is_readable') def __init__(self, doc): """Parse an ismaster response from the server.""" self._server_type = _get_server_type(doc) self._doc = doc self._is_writable = self._server_type in ( SERVER_TYPE.RSPrimary, SERVER_TYPE.Standalone, SERVER_TYPE.Mongos) self._is_readable = ( self.server_type == SERVER_TYPE.RSSecondary or self._is_writable) @property def document(self): """The complete ismaster command response document. .. versionadded:: 3.4 """ return self._doc.copy() @property def server_type(self): return self._server_type @property def all_hosts(self): """List of hosts, passives, and arbiters known to this server.""" return set(imap(common.clean_node, itertools.chain( self._doc.get('hosts', []), self._doc.get('passives', []), self._doc.get('arbiters', [])))) @property def tags(self): """Replica set member tags or empty dict.""" return self._doc.get('tags', {}) @property def primary(self): """This server's opinion about who the primary is, or None.""" if self._doc.get('primary'): return common.partition_node(self._doc['primary']) else: return None @property def replica_set_name(self): """Replica set name or None.""" return self._doc.get('setName') @property def max_bson_size(self): return self._doc.get('maxBsonObjectSize', common.MAX_BSON_SIZE) @property def max_message_size(self): return self._doc.get('maxMessageSizeBytes', 2 * self.max_bson_size) @property def max_write_batch_size(self): return self._doc.get('maxWriteBatchSize', common.MAX_WRITE_BATCH_SIZE) @property def min_wire_version(self): return self._doc.get('minWireVersion', common.MIN_WIRE_VERSION) @property def max_wire_version(self): return self._doc.get('maxWireVersion', common.MAX_WIRE_VERSION) @property def set_version(self): return self._doc.get('setVersion') @property def election_id(self): return self._doc.get('electionId') @property def cluster_time(self): return self._doc.get('$clusterTime') @property def logical_session_timeout_minutes(self): return self._doc.get('logicalSessionTimeoutMinutes') @property def is_writable(self): return self._is_writable @property def is_readable(self): return self._is_readable @property def me(self): me = self._doc.get('me') if me: return common.clean_node(me) @property def last_write_date(self): return self._doc.get('lastWrite', {}).get('lastWriteDate') pymongo-3.6.1/pymongo/mongo_replica_set_client.py0000644000076600000240000000364313245617773022551 0ustar shanestaff00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Deprecated. See :doc:`/examples/high_availability`.""" import warnings from pymongo import mongo_client class MongoReplicaSetClient(mongo_client.MongoClient): """Deprecated alias for :class:`~pymongo.mongo_client.MongoClient`. :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` will be removed in a future version of PyMongo. .. versionchanged:: 3.0 :class:`~pymongo.mongo_client.MongoClient` is now the one and only client class for a standalone server, mongos, or replica set. It includes the functionality that had been split into :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`: it can connect to a replica set, discover all its members, and monitor the set for stepdowns, elections, and reconfigs. The ``refresh`` method is removed from :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`, as are the ``seeds`` and ``hosts`` properties. """ def __init__(self, *args, **kwargs): warnings.warn('MongoReplicaSetClient is deprecated, use MongoClient' ' to connect to a replica set', DeprecationWarning, stacklevel=2) super(MongoReplicaSetClient, self).__init__(*args, **kwargs) def __repr__(self): return "MongoReplicaSetClient(%s)" % (self._repr_helper(),) pymongo-3.6.1/pymongo/_cmessagemodule.c0000644000076600000240000012547013245621354020441 0ustar shanestaff00000000000000/* * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains C implementations of some of the functions * needed by the message module. If possible, these implementations * should be used to speed up message creation. */ #include "Python.h" #include "_cbsonmodule.h" #include "buffer.h" struct module_state { PyObject* _cbson; }; /* See comments about module initialization in _cbsonmodule.c */ #if PY_MAJOR_VERSION >= 3 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) #else #define GETSTATE(m) (&_state) static struct module_state _state; #endif #if PY_MAJOR_VERSION >= 3 #define BYTES_FORMAT_STRING "y#" #else #define BYTES_FORMAT_STRING "s#" #endif #define DOC_TOO_LARGE_FMT "BSON document too large (%d bytes)" \ " - the connected server supports" \ " BSON document sizes up to %ld bytes." /* Get an error class from the pymongo.errors module. * * Returns a new ref */ static PyObject* _error(char* name) { PyObject* error; PyObject* errors = PyImport_ImportModule("pymongo.errors"); if (!errors) { return NULL; } error = PyObject_GetAttrString(errors, name); Py_DECREF(errors); return error; } /* add a lastError message on the end of the buffer. * returns 0 on failure */ static int add_last_error(PyObject* self, buffer_t buffer, int request_id, char* ns, int nslen, codec_options_t* options, PyObject* args) { struct module_state *state = GETSTATE(self); int message_start; int document_start; int message_length; int document_length; PyObject* key = NULL; PyObject* value = NULL; Py_ssize_t pos = 0; PyObject* one; char *p = strchr(ns, '.'); /* Length of the database portion of ns. */ nslen = p ? (int)(p - ns) : nslen; message_start = buffer_save_space(buffer, 4); if (message_start == -1) { PyErr_NoMemory(); return 0; } if (!buffer_write_int32(buffer, (int32_t)request_id) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" /* responseTo */ "\xd4\x07\x00\x00" /* opcode */ "\x00\x00\x00\x00", /* options */ 12) || !buffer_write_bytes(buffer, ns, nslen) || /* database */ !buffer_write_bytes(buffer, ".$cmd\x00" /* collection name */ "\x00\x00\x00\x00" /* skip */ "\xFF\xFF\xFF\xFF", /* limit (-1) */ 14)) { return 0; } /* save space for length */ document_start = buffer_save_space(buffer, 4); if (document_start == -1) { PyErr_NoMemory(); return 0; } /* getlasterror: 1 */ if (!(one = PyLong_FromLong(1))) return 0; if (!write_pair(state->_cbson, buffer, "getlasterror", 12, one, 0, options, 1)) { Py_DECREF(one); return 0; } Py_DECREF(one); /* getlasterror options */ while (PyDict_Next(args, &pos, &key, &value)) { if (!decode_and_write_pair(state->_cbson, buffer, key, value, 0, options, 0)) { return 0; } } /* EOD */ if (!buffer_write_bytes(buffer, "\x00", 1)) { return 0; } message_length = buffer_get_position(buffer) - message_start; document_length = buffer_get_position(buffer) - document_start; buffer_write_int32_at_position( buffer, message_start, (int32_t)message_length); buffer_write_int32_at_position( buffer, document_start, (int32_t)document_length); return 1; } static int init_insert_buffer(buffer_t buffer, int request_id, int options, const char* coll_name, int coll_name_len) { /* Save space for message length */ int length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyErr_NoMemory(); return length_location; } if (!buffer_write_int32(buffer, (int32_t)request_id) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" "\xd2\x07\x00\x00", 8) || !buffer_write_int32(buffer, (int32_t)options) || !buffer_write_bytes(buffer, coll_name, coll_name_len + 1)) { return -1; } return length_location; } static PyObject* _cbson_insert_message(PyObject* self, PyObject* args) { /* Used by the Bulk API to insert into pre-2.6 servers. Collection.insert * uses _cbson_do_batched_insert. */ struct module_state *state = GETSTATE(self); /* NOTE just using a random number as the request_id */ int request_id = rand(); char* collection_name = NULL; int collection_name_length; PyObject* docs; PyObject* doc; PyObject* iterator; int before, cur_size, max_size = 0; int flags = 0; unsigned char check_keys; unsigned char safe; unsigned char continue_on_error; codec_options_t options; PyObject* last_error_args; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#ObbObO&", "utf-8", &collection_name, &collection_name_length, &docs, &check_keys, &safe, &last_error_args, &continue_on_error, convert_codec_options, &options)) { return NULL; } if (continue_on_error) { flags += 1; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } length_location = init_insert_buffer(buffer, request_id, flags, collection_name, collection_name_length); if (length_location == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); buffer_free(buffer); return NULL; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } while ((doc = PyIter_Next(iterator)) != NULL) { before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { Py_DECREF(doc); Py_DECREF(iterator); destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } Py_DECREF(doc); cur_size = buffer_get_position(buffer) - before; max_size = (cur_size > max_size) ? cur_size : max_size; } Py_DECREF(iterator); if (PyErr_Occurred()) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } if (!max_size) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert"); Py_DECREF(InvalidOperation); } destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } message_length = buffer_get_position(buffer) - length_location; buffer_write_int32_at_position( buffer, length_location, (int32_t)message_length); if (safe) { if (!add_last_error(self, buffer, request_id, collection_name, collection_name_length, &options, last_error_args)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } } PyMem_Free(collection_name); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); destroy_codec_options(&options); buffer_free(buffer); return result; } static PyObject* _cbson_update_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ struct module_state *state = GETSTATE(self); int request_id = rand(); char* collection_name = NULL; int collection_name_length; int before, cur_size, max_size = 0; PyObject* doc; PyObject* spec; unsigned char multi; unsigned char upsert; unsigned char safe; unsigned char check_keys; codec_options_t options; PyObject* last_error_args; int flags; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#bbOObObO&", "utf-8", &collection_name, &collection_name_length, &upsert, &multi, &spec, &doc, &safe, &last_error_args, &check_keys, convert_codec_options, &options)) { return NULL; } flags = 0; if (upsert) { flags += 1; } if (multi) { flags += 2; } buffer = buffer_new(); if (!buffer) { destroy_codec_options(&options); PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } if (!buffer_write_int32(buffer, (int32_t)request_id) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" "\xd1\x07\x00\x00" "\x00\x00\x00\x00", 12) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1) || !buffer_write_int32(buffer, (int32_t)flags)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, spec, 0, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } max_size = buffer_get_position(buffer) - before; before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } cur_size = buffer_get_position(buffer) - before; max_size = (cur_size > max_size) ? cur_size : max_size; message_length = buffer_get_position(buffer) - length_location; buffer_write_int32_at_position( buffer, length_location, (int32_t)message_length); if (safe) { if (!add_last_error(self, buffer, request_id, collection_name, collection_name_length, &options, last_error_args)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } } PyMem_Free(collection_name); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); destroy_codec_options(&options); buffer_free(buffer); return result; } static PyObject* _cbson_query_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ struct module_state *state = GETSTATE(self); int request_id = rand(); PyObject* cluster_time = NULL; unsigned int flags; char* collection_name = NULL; int collection_name_length; int begin, cur_size, max_size = 0; int num_to_skip; int num_to_return; PyObject* query; PyObject* field_selector; codec_options_t options; buffer_t buffer; int length_location, message_length; unsigned char check_keys = 0; PyObject* result; if (!PyArg_ParseTuple(args, "Iet#iiOOO&|b", &flags, "utf-8", &collection_name, &collection_name_length, &num_to_skip, &num_to_return, &query, &field_selector, convert_codec_options, &options, &check_keys)) { return NULL; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } /* Pop $clusterTime from dict and write it at the end, avoiding an error * from the $-prefix and check_keys. * * If "dict" is a defaultdict we don't want to call PyMapping_GetItemString * on it. That would **create** an _id where one didn't previously exist * (PYTHON-871). */ if (PyDict_Check(query)) { cluster_time = PyDict_GetItemString(query, "$clusterTime"); if (cluster_time) { /* PyDict_GetItemString returns a borrowed reference. */ Py_INCREF(cluster_time); if (-1 == PyMapping_DelItemString(query, "$clusterTime")) { destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } } } else if (PyMapping_HasKeyString(query, "$clusterTime")) { cluster_time = PyMapping_GetItemString(query, "$clusterTime"); if (!cluster_time || -1 == PyMapping_DelItemString(query, "$clusterTime")) { destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } } if (!buffer_write_int32(buffer, (int32_t)request_id) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00\xd4\x07\x00\x00", 8) || !buffer_write_int32(buffer, (int32_t)flags) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1) || !buffer_write_int32(buffer, (int32_t)num_to_skip) || !buffer_write_int32(buffer, (int32_t)num_to_return)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); Py_XDECREF(cluster_time); return NULL; } begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, query, check_keys, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); Py_XDECREF(cluster_time); return NULL; } /* back up a byte and write $clusterTime */ if (cluster_time) { int length; char zero = 0; buffer_update_position(buffer, buffer_get_position(buffer) - 1); if (!write_pair(state->_cbson, buffer, "$clusterTime", 12, cluster_time, 0, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); Py_DECREF(cluster_time); return NULL; } if (!buffer_write_bytes(buffer, &zero, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); Py_DECREF(cluster_time); return NULL; } length = buffer_get_position(buffer) - begin; buffer_write_int32_at_position(buffer, begin, (int32_t)length); /* undo popping $clusterTime */ if (-1 == PyMapping_SetItemString( query, "$clusterTime", cluster_time)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); Py_DECREF(cluster_time); return NULL; } Py_DECREF(cluster_time); } max_size = buffer_get_position(buffer) - begin; if (field_selector != Py_None) { begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, field_selector, 0, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } cur_size = buffer_get_position(buffer) - begin; max_size = (cur_size > max_size) ? cur_size : max_size; } PyMem_Free(collection_name); message_length = buffer_get_position(buffer) - length_location; buffer_write_int32_at_position( buffer, length_location, (int32_t)message_length); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); destroy_codec_options(&options); buffer_free(buffer); return result; } static PyObject* _cbson_get_more_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ int request_id = rand(); char* collection_name = NULL; int collection_name_length; int num_to_return; long long cursor_id; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#iL", "utf-8", &collection_name, &collection_name_length, &num_to_return, &cursor_id)) { return NULL; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } if (!buffer_write_int32(buffer, (int32_t)request_id) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" "\xd5\x07\x00\x00" "\x00\x00\x00\x00", 12) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1) || !buffer_write_int32(buffer, (int32_t)num_to_return) || !buffer_write_int64(buffer, (int64_t)cursor_id)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } PyMem_Free(collection_name); message_length = buffer_get_position(buffer) - length_location; buffer_write_int32_at_position( buffer, length_location, (int32_t)message_length); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING, request_id, buffer_get_buffer(buffer), buffer_get_position(buffer)); buffer_free(buffer); return result; } static void _set_document_too_large(int size, long max) { PyObject* DocumentTooLarge = _error("DocumentTooLarge"); if (DocumentTooLarge) { #if PY_MAJOR_VERSION >= 3 PyObject* error = PyUnicode_FromFormat(DOC_TOO_LARGE_FMT, size, max); #else PyObject* error = PyString_FromFormat(DOC_TOO_LARGE_FMT, size, max); #endif if (error) { PyErr_SetObject(DocumentTooLarge, error); Py_DECREF(error); } Py_DECREF(DocumentTooLarge); } } static PyObject* _send_insert(PyObject* self, PyObject* ctx, PyObject* gle_args, buffer_t buffer, char* coll_name, int coll_len, int request_id, int safe, codec_options_t* options, PyObject* to_publish) { if (safe) { if (!add_last_error(self, buffer, request_id, coll_name, coll_len, options, gle_args)) { return NULL; } } /* The max_doc_size parameter for legacy_write is the max size of any * document in buffer. We enforced max size already, pass 0 here. */ return PyObject_CallMethod(ctx, "legacy_write", "i" BYTES_FORMAT_STRING "iNO", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), 0, PyBool_FromLong((long)safe), to_publish); } static PyObject* _cbson_do_batched_insert(PyObject* self, PyObject* args) { struct module_state *state = GETSTATE(self); /* NOTE just using a random number as the request_id */ int request_id = rand(); int send_safe, flags = 0; int length_location, message_length; int collection_name_length; char* collection_name = NULL; PyObject* docs; PyObject* doc; PyObject* iterator; PyObject* ctx; PyObject* last_error_args; PyObject* result; PyObject* max_bson_size_obj; PyObject* max_message_size_obj; PyObject* to_publish = NULL; unsigned char check_keys; unsigned char safe; unsigned char continue_on_error; codec_options_t options; unsigned char empty = 1; long max_bson_size; long max_message_size; buffer_t buffer; PyObject *exc_type = NULL, *exc_value = NULL, *exc_trace = NULL; if (!PyArg_ParseTuple(args, "et#ObbObO&O", "utf-8", &collection_name, &collection_name_length, &docs, &check_keys, &safe, &last_error_args, &continue_on_error, convert_codec_options, &options, &ctx)) { return NULL; } if (continue_on_error) { flags += 1; } /* * If we are doing unacknowledged writes *and* continue_on_error * is True it's pointless (and slower) to send GLE. */ send_safe = (safe || !continue_on_error); max_bson_size_obj = PyObject_GetAttrString(ctx, "max_bson_size"); #if PY_MAJOR_VERSION >= 3 max_bson_size = PyLong_AsLong(max_bson_size_obj); #else max_bson_size = PyInt_AsLong(max_bson_size_obj); #endif Py_XDECREF(max_bson_size_obj); if (max_bson_size == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } max_message_size_obj = PyObject_GetAttrString(ctx, "max_message_size"); #if PY_MAJOR_VERSION >= 3 max_message_size = PyLong_AsLong(max_message_size_obj); #else max_message_size = PyInt_AsLong(max_message_size_obj); #endif Py_XDECREF(max_message_size_obj); if (max_message_size == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } buffer = buffer_new(); if (!buffer) { destroy_codec_options(&options); PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } length_location = init_insert_buffer(buffer, request_id, flags, collection_name, collection_name_length); if (length_location == -1) { goto insertfail; } if (!(to_publish = PyList_New(0))) { goto insertfail; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } goto insertfail; } while ((doc = PyIter_Next(iterator)) != NULL) { int before = buffer_get_position(buffer); int cur_size; if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { goto iterfail; } cur_size = buffer_get_position(buffer) - before; if (cur_size > max_bson_size) { /* If we've encoded anything send it before raising. */ if (!empty) { buffer_update_position(buffer, before); message_length = buffer_get_position(buffer) - length_location; buffer_write_int32_at_position( buffer, length_location, (int32_t)message_length); result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, send_safe, &options, to_publish); if (!result) goto iterfail; Py_DECREF(result); } _set_document_too_large(cur_size, max_bson_size); goto iterfail; } empty = 0; /* We have enough data, send this batch. */ if (buffer_get_position(buffer) > max_message_size) { int new_request_id = rand(); int message_start; buffer_t new_buffer = buffer_new(); if (!new_buffer) { PyErr_NoMemory(); goto iterfail; } message_start = init_insert_buffer(new_buffer, new_request_id, flags, collection_name, collection_name_length); if (message_start == -1) { buffer_free(new_buffer); goto iterfail; } /* Copy the overflow encoded document into the new buffer. */ if (!buffer_write_bytes(new_buffer, (const char*)buffer_get_buffer(buffer) + before, cur_size)) { buffer_free(new_buffer); goto iterfail; } /* Roll back to the beginning of this document. */ buffer_update_position(buffer, before); message_length = buffer_get_position(buffer) - length_location; buffer_write_int32_at_position( buffer, length_location, (int32_t)message_length); result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, send_safe, &options, to_publish); buffer_free(buffer); buffer = new_buffer; request_id = new_request_id; length_location = message_start; Py_DECREF(to_publish); if (!(to_publish = PyList_New(0))) { goto insertfail; } if (!result) { PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; PyObject* OperationFailure; PyErr_Fetch(&etype, &evalue, &etrace); OperationFailure = _error("OperationFailure"); if (OperationFailure) { if (PyErr_GivenExceptionMatches(etype, OperationFailure)) { if (!safe || continue_on_error) { Py_DECREF(OperationFailure); if (!safe) { /* We're doing unacknowledged writes and * continue_on_error is False. Just return. */ Py_DECREF(etype); Py_XDECREF(evalue); Py_XDECREF(etrace); Py_DECREF(to_publish); Py_DECREF(iterator); Py_DECREF(doc); buffer_free(buffer); PyMem_Free(collection_name); Py_RETURN_NONE; } /* continue_on_error is True, store the error * details to re-raise after the final batch */ Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); exc_type = etype; exc_value = evalue; exc_trace = etrace; if (PyList_Append(to_publish, doc) < 0) { goto iterfail; } Py_CLEAR(doc); continue; } } Py_DECREF(OperationFailure); } /* This isn't OperationFailure, we couldn't * import OperationFailure, or we are doing * acknowledged writes. Re-raise immediately. */ PyErr_Restore(etype, evalue, etrace); goto iterfail; } else { Py_DECREF(result); } } if (PyList_Append(to_publish, doc) < 0) { goto iterfail; } Py_CLEAR(doc); } Py_DECREF(iterator); if (PyErr_Occurred()) { goto insertfail; } if (empty) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert"); Py_DECREF(InvalidOperation); } goto insertfail; } message_length = buffer_get_position(buffer) - length_location; buffer_write_int32_at_position( buffer, length_location, (int32_t)message_length); /* Send the last (or only) batch */ result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, safe, &options, to_publish); Py_DECREF(to_publish); PyMem_Free(collection_name); buffer_free(buffer); if (!result) { Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); return NULL; } else { Py_DECREF(result); } if (exc_type) { /* Re-raise any previously stored exception * due to continue_on_error being True */ PyErr_Restore(exc_type, exc_value, exc_trace); return NULL; } Py_RETURN_NONE; iterfail: Py_XDECREF(doc); Py_DECREF(iterator); insertfail: Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); Py_XDECREF(to_publish); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } static buffer_t _command_buffer_new(char* ns, int ns_len) { buffer_t buffer; if (!(buffer = buffer_new())) { PyErr_NoMemory(); return NULL; } /* Save space for message length and request id */ if ((buffer_save_space(buffer, 8)) == -1) { PyErr_NoMemory(); buffer_free(buffer); return NULL; } if (!buffer_write_bytes(buffer, "\x00\x00\x00\x00" /* responseTo */ "\xd4\x07\x00\x00" /* opcode */ "\x00\x00\x00\x00", /* flags */ 12) || !buffer_write_bytes(buffer, ns, ns_len + 1) || /* namespace */ !buffer_write_bytes(buffer, "\x00\x00\x00\x00" /* skip */ "\xFF\xFF\xFF\xFF", /* limit (-1) */ 8)) { buffer_free(buffer); return NULL; } return buffer; } #define _INSERT 0 #define _UPDATE 1 #define _DELETE 2 static PyObject* _cbson_do_batched_write_command(PyObject* self, PyObject* args) { struct module_state *state = GETSTATE(self); long max_bson_size; long max_cmd_size; long max_write_batch_size; int idx = 0; int cmd_len_loc; int lst_len_loc; int ns_len; int request_id; int position; int length; char *ns = NULL; PyObject* max_bson_size_obj; PyObject* max_write_batch_size_obj; PyObject* command; PyObject* doc; PyObject* docs; PyObject* ctx; PyObject* iterator; PyObject* result; PyObject* to_publish = NULL; unsigned char op; unsigned char check_keys; codec_options_t options; buffer_t buffer; if (!PyArg_ParseTuple(args, "et#bOObO&O", "utf-8", &ns, &ns_len, &op, &command, &docs, &check_keys, convert_codec_options, &options, &ctx)) { return NULL; } max_bson_size_obj = PyObject_GetAttrString(ctx, "max_bson_size"); #if PY_MAJOR_VERSION >= 3 max_bson_size = PyLong_AsLong(max_bson_size_obj); #else max_bson_size = PyInt_AsLong(max_bson_size_obj); #endif Py_XDECREF(max_bson_size_obj); if (max_bson_size == -1) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } /* * Max BSON object size + 16k - 2 bytes for ending NUL bytes * XXX: This should come from the server - SERVER-10643 */ max_cmd_size = max_bson_size + 16382; max_write_batch_size_obj = PyObject_GetAttrString(ctx, "max_write_batch_size"); #if PY_MAJOR_VERSION >= 3 max_write_batch_size = PyLong_AsLong(max_write_batch_size_obj); #else max_write_batch_size = PyInt_AsLong(max_write_batch_size_obj); #endif Py_XDECREF(max_write_batch_size_obj); if (max_write_batch_size == -1) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } if (!(to_publish = PyList_New(0))) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } if (!(buffer = _command_buffer_new(ns, ns_len))) { destroy_codec_options(&options); PyMem_Free(ns); Py_DECREF(to_publish); return NULL; } PyMem_Free(ns); /* Position of command document length */ cmd_len_loc = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, command, 0, &options, 0)) { goto cmdfail; } /* Write type byte for array */ *(buffer_get_buffer(buffer) + (buffer_get_position(buffer) - 1)) = 0x4; switch (op) { case _INSERT: { if (!buffer_write_bytes(buffer, "documents\x00", 10)) goto cmdfail; break; } case _UPDATE: { /* MongoDB does key validation for update. */ check_keys = 0; if (!buffer_write_bytes(buffer, "updates\x00", 8)) goto cmdfail; break; } case _DELETE: { /* Never check keys in a delete command. */ check_keys = 0; if (!buffer_write_bytes(buffer, "deletes\x00", 8)) goto cmdfail; break; } default: { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "Unknown command"); Py_DECREF(InvalidOperation); } goto cmdfail; } } /* Save space for list document */ lst_len_loc = buffer_save_space(buffer, 4); if (lst_len_loc == -1) { PyErr_NoMemory(); goto cmdfail; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } goto cmdfail; } while ((doc = PyIter_Next(iterator)) != NULL) { int sub_doc_begin = buffer_get_position(buffer); int cur_doc_begin; int cur_size; int enough_data = 0; int enough_documents = 0; char key[16]; INT2STRING(key, idx); if (!buffer_write_bytes(buffer, "\x03", 1) || !buffer_write_bytes(buffer, key, (int)strlen(key) + 1)) { goto cmditerfail; } cur_doc_begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { goto cmditerfail; } /* We have enough data, return this batch. * max_cmd_size accounts for the two trailing null bytes. */ enough_data = (buffer_get_position(buffer) > max_cmd_size); enough_documents = (idx >= max_write_batch_size); if (enough_data || enough_documents) { cur_size = buffer_get_position(buffer) - cur_doc_begin; /* This single document is too large for the command. */ if (!idx) { if (op == _INSERT) { _set_document_too_large(cur_size, max_bson_size); } else { PyObject* DocumentTooLarge = _error("DocumentTooLarge"); if (DocumentTooLarge) { /* * There's nothing intelligent we can say * about size for update and remove. */ PyErr_SetString(DocumentTooLarge, "command document too large"); Py_DECREF(DocumentTooLarge); } } goto cmditerfail; } /* * Roll the existing buffer back to the beginning * of the last document encoded. */ buffer_update_position(buffer, sub_doc_begin); break; } if (PyList_Append(to_publish, doc) < 0) { goto cmditerfail; } Py_CLEAR(doc); idx += 1; } Py_DECREF(iterator); if (PyErr_Occurred()) { goto cmdfail; } if (!buffer_write_bytes(buffer, "\x00\x00", 2)) goto cmdfail; request_id = rand(); position = buffer_get_position(buffer); length = position - lst_len_loc - 1; buffer_write_int32_at_position(buffer, lst_len_loc, (int32_t)length); length = position - cmd_len_loc; buffer_write_int32_at_position(buffer, cmd_len_loc, (int32_t)length); buffer_write_int32_at_position(buffer, 0, (int32_t)position); buffer_write_int32_at_position(buffer, 4, (int32_t)request_id); result = Py_BuildValue("i" BYTES_FORMAT_STRING "O", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), to_publish); Py_DECREF(to_publish); buffer_free(buffer); destroy_codec_options(&options); return result; cmditerfail: Py_XDECREF(doc); Py_DECREF(iterator); cmdfail: destroy_codec_options(&options); Py_XDECREF(to_publish); buffer_free(buffer); return NULL; } static PyMethodDef _CMessageMethods[] = { {"_insert_message", _cbson_insert_message, METH_VARARGS, "Create an insert message to be sent to MongoDB"}, {"_update_message", _cbson_update_message, METH_VARARGS, "create an update message to be sent to MongoDB"}, {"_query_message", _cbson_query_message, METH_VARARGS, "create a query message to be sent to MongoDB"}, {"_get_more_message", _cbson_get_more_message, METH_VARARGS, "create a get more message to be sent to MongoDB"}, {"_do_batched_insert", _cbson_do_batched_insert, METH_VARARGS, "insert a batch of documents, splitting the batch as needed"}, {"_do_batched_write_command", _cbson_do_batched_write_command, METH_VARARGS, "Create the next batched insert, update, or delete command"}, {NULL, NULL, 0, NULL} }; #if PY_MAJOR_VERSION >= 3 #define INITERROR return NULL static int _cmessage_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->_cbson); return 0; } static int _cmessage_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->_cbson); return 0; } static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_cmessage", NULL, sizeof(struct module_state), _CMessageMethods, NULL, _cmessage_traverse, _cmessage_clear, NULL }; PyMODINIT_FUNC PyInit__cmessage(void) #else #define INITERROR return PyMODINIT_FUNC init_cmessage(void) #endif { PyObject *_cbson = NULL; PyObject *c_api_object = NULL; PyObject *m = NULL; /* Store a reference to the _cbson module since it's needed to call some * of its functions */ _cbson = PyImport_ImportModule("bson._cbson"); if (_cbson == NULL) { goto fail; } /* Import C API of _cbson * The header file accesses _cbson_API to call the functions */ c_api_object = PyObject_GetAttrString(_cbson, "_C_API"); if (c_api_object == NULL) { goto fail; } #if PY_VERSION_HEX >= 0x03010000 _cbson_API = (void **)PyCapsule_GetPointer(c_api_object, "_cbson._C_API"); #else _cbson_API = (void **)PyCObject_AsVoidPtr(c_api_object); #endif if (_cbson_API == NULL) { goto fail; } #if PY_MAJOR_VERSION >= 3 /* Returns a new reference. */ m = PyModule_Create(&moduledef); #else /* Returns a borrowed reference. */ m = Py_InitModule("_cmessage", _CMessageMethods); #endif if (m == NULL) { goto fail; } GETSTATE(m)->_cbson = _cbson; Py_DECREF(c_api_object); #if PY_MAJOR_VERSION >= 3 return m; #else return; #endif fail: #if PY_MAJOR_VERSION >= 3 Py_XDECREF(m); #endif Py_XDECREF(c_api_object); Py_XDECREF(_cbson); INITERROR; } pymongo-3.6.1/pymongo/server_description.py0000644000076600000240000001432013245621354021413 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Represent one server the driver is connected to.""" from bson import EPOCH_NAIVE from pymongo.server_type import SERVER_TYPE from pymongo.ismaster import IsMaster from pymongo.monotonic import time as _time def _total_seconds(delta): """Total seconds in the duration.""" if hasattr(delta, 'total_seconds'): return delta.total_seconds() # Python 2.6. return ((delta.days * 86400 + delta.seconds) * 10 ** 6 + delta.microseconds) / 10.0 ** 6 class ServerDescription(object): """Immutable representation of one server. :Parameters: - `address`: A (host, port) pair - `ismaster`: Optional IsMaster instance - `round_trip_time`: Optional float - `error`: Optional, the last error attempting to connect to the server """ __slots__ = ( '_address', '_server_type', '_all_hosts', '_tags', '_replica_set_name', '_primary', '_max_bson_size', '_max_message_size', '_max_write_batch_size', '_min_wire_version', '_max_wire_version', '_round_trip_time', '_me', '_is_writable', '_is_readable', '_ls_timeout_minutes', '_error', '_set_version', '_election_id', '_cluster_time', '_last_write_date', '_last_update_time') def __init__( self, address, ismaster=None, round_trip_time=None, error=None): self._address = address if not ismaster: ismaster = IsMaster({}) self._server_type = ismaster.server_type self._all_hosts = ismaster.all_hosts self._tags = ismaster.tags self._replica_set_name = ismaster.replica_set_name self._primary = ismaster.primary self._max_bson_size = ismaster.max_bson_size self._max_message_size = ismaster.max_message_size self._max_write_batch_size = ismaster.max_write_batch_size self._min_wire_version = ismaster.min_wire_version self._max_wire_version = ismaster.max_wire_version self._set_version = ismaster.set_version self._election_id = ismaster.election_id self._cluster_time = ismaster.cluster_time self._is_writable = ismaster.is_writable self._is_readable = ismaster.is_readable self._ls_timeout_minutes = ismaster.logical_session_timeout_minutes self._round_trip_time = round_trip_time self._me = ismaster.me self._last_update_time = _time() self._error = error if ismaster.last_write_date: # Convert from datetime to seconds. delta = ismaster.last_write_date - EPOCH_NAIVE self._last_write_date = _total_seconds(delta) else: self._last_write_date = None @property def address(self): """The address (host, port) of this server.""" return self._address @property def server_type(self): """The type of this server.""" return self._server_type @property def server_type_name(self): """The server type as a human readable string. .. versionadded:: 3.4 """ return SERVER_TYPE._fields[self._server_type] @property def all_hosts(self): """List of hosts, passives, and arbiters known to this server.""" return self._all_hosts @property def tags(self): return self._tags @property def replica_set_name(self): """Replica set name or None.""" return self._replica_set_name @property def primary(self): """This server's opinion about who the primary is, or None.""" return self._primary @property def max_bson_size(self): return self._max_bson_size @property def max_message_size(self): return self._max_message_size @property def max_write_batch_size(self): return self._max_write_batch_size @property def min_wire_version(self): return self._min_wire_version @property def max_wire_version(self): return self._max_wire_version @property def set_version(self): return self._set_version @property def election_id(self): return self._election_id @property def cluster_time(self): return self._cluster_time @property def election_tuple(self): return self._set_version, self._election_id @property def me(self): return self._me @property def logical_session_timeout_minutes(self): return self._ls_timeout_minutes @property def last_write_date(self): return self._last_write_date @property def last_update_time(self): return self._last_update_time @property def round_trip_time(self): """The current average latency or None.""" # This override is for unittesting only! if self._address in self._host_to_round_trip_time: return self._host_to_round_trip_time[self._address] return self._round_trip_time @property def error(self): """The last error attempting to connect to the server, or None.""" return self._error @property def is_writable(self): return self._is_writable @property def is_readable(self): return self._is_readable @property def is_server_type_known(self): return self.server_type != SERVER_TYPE.Unknown @property def retryable_writes_supported(self): """Checks if this server supports retryable writes.""" return ( self._ls_timeout_minutes is not None and self._server_type in (SERVER_TYPE.Mongos, SERVER_TYPE.RSPrimary)) # For unittesting only. Use under no circumstances! _host_to_round_trip_time = {} pymongo-3.6.1/pymongo/__init__.py0000644000076600000240000000540513246100114017230 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Python driver for MongoDB.""" ASCENDING = 1 """Ascending sort order.""" DESCENDING = -1 """Descending sort order.""" GEO2D = "2d" """Index specifier for a 2-dimensional `geospatial index`_. .. _geospatial index: http://docs.mongodb.org/manual/core/2d/ """ GEOHAYSTACK = "geoHaystack" """Index specifier for a 2-dimensional `haystack index`_. .. versionadded:: 2.1 .. _haystack index: http://docs.mongodb.org/manual/core/geohaystack/ """ GEOSPHERE = "2dsphere" """Index specifier for a `spherical geospatial index`_. .. versionadded:: 2.5 .. _spherical geospatial index: http://docs.mongodb.org/manual/core/2dsphere/ """ HASHED = "hashed" """Index specifier for a `hashed index`_. .. versionadded:: 2.5 .. _hashed index: http://docs.mongodb.org/manual/core/index-hashed/ """ TEXT = "text" """Index specifier for a `text index`_. .. versionadded:: 2.7.1 .. _text index: http://docs.mongodb.org/manual/core/index-text/ """ OFF = 0 """No database profiling.""" SLOW_ONLY = 1 """Only profile slow operations.""" ALL = 2 """Profile all operations.""" version_tuple = (3, 6, 1) def get_version_string(): if isinstance(version_tuple[-1], str): return '.'.join(map(str, version_tuple[:-1])) + version_tuple[-1] return '.'.join(map(str, version_tuple)) __version__ = version = get_version_string() """Current version of PyMongo.""" from pymongo.collection import ReturnDocument from pymongo.common import (MIN_SUPPORTED_WIRE_VERSION, MAX_SUPPORTED_WIRE_VERSION) from pymongo.cursor import CursorType from pymongo.mongo_client import MongoClient from pymongo.mongo_replica_set_client import MongoReplicaSetClient from pymongo.operations import (IndexModel, InsertOne, DeleteOne, DeleteMany, UpdateOne, UpdateMany, ReplaceOne) from pymongo.read_preferences import ReadPreference from pymongo.write_concern import WriteConcern def has_c(): """Is the C extension installed?""" try: from pymongo import _cmessage return True except ImportError: return False pymongo-3.6.1/pymongo/message.py0000644000076600000240000010600713245621354017132 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for creating `messages `_ to be sent to MongoDB. .. note:: This module is for internal use and is generally not needed by application developers. """ import datetime import random import struct import bson from bson import CodecOptions from bson.codec_options import DEFAULT_CODEC_OPTIONS from bson.py3compat import b, StringIO from bson.son import SON try: from pymongo import _cmessage _use_c = True except ImportError: _use_c = False from pymongo.errors import (ConfigurationError, CursorNotFound, DocumentTooLarge, ExecutionTimeout, InvalidOperation, NotMasterError, OperationFailure, ProtocolError) from pymongo.read_concern import DEFAULT_READ_CONCERN from pymongo.read_preferences import ReadPreference MAX_INT32 = 2147483647 MIN_INT32 = -2147483648 # Overhead allowed for encoded command documents. _COMMAND_OVERHEAD = 16382 _INSERT = 0 _UPDATE = 1 _DELETE = 2 _EMPTY = b'' _BSONOBJ = b'\x03' _ZERO_8 = b'\x00' _ZERO_16 = b'\x00\x00' _ZERO_32 = b'\x00\x00\x00\x00' _ZERO_64 = b'\x00\x00\x00\x00\x00\x00\x00\x00' _SKIPLIM = b'\x00\x00\x00\x00\xff\xff\xff\xff' _OP_MAP = { _INSERT: b'\x04documents\x00\x00\x00\x00\x00', _UPDATE: b'\x04updates\x00\x00\x00\x00\x00', _DELETE: b'\x04deletes\x00\x00\x00\x00\x00', } _UJOIN = u"%s.%s" _UNICODE_REPLACE_CODEC_OPTIONS = CodecOptions( unicode_decode_error_handler='replace') def _randint(): """Generate a pseudo random 32 bit integer.""" return random.randint(MIN_INT32, MAX_INT32) def _maybe_add_read_preference(spec, read_preference): """Add $readPreference to spec when appropriate.""" mode = read_preference.mode tag_sets = read_preference.tag_sets max_staleness = read_preference.max_staleness # Only add $readPreference if it's something other than primary to avoid # problems with mongos versions that don't support read preferences. Also, # for maximum backwards compatibility, don't add $readPreference for # secondaryPreferred unless tags or maxStalenessSeconds are in use (setting # the slaveOkay bit has the same effect). if mode and ( mode != ReadPreference.SECONDARY_PREFERRED.mode or tag_sets != [{}] or max_staleness != -1): if "$query" not in spec: spec = SON([("$query", spec)]) spec["$readPreference"] = read_preference.document return spec def _convert_exception(exception): """Convert an Exception into a failure document for publishing.""" return {'errmsg': str(exception), 'errtype': exception.__class__.__name__} def _convert_write_result(operation, command, result): """Convert a legacy write result to write commmand format.""" # Based on _merge_legacy from bulk.py affected = result.get("n", 0) res = {"ok": 1, "n": affected} errmsg = result.get("errmsg", result.get("err", "")) if errmsg: # The write was successful on at least the primary so don't return. if result.get("wtimeout"): res["writeConcernError"] = {"errmsg": errmsg, "code": 64, "errInfo": {"wtimeout": True}} else: # The write failed. error = {"index": 0, "code": result.get("code", 8), "errmsg": errmsg} if "errInfo" in result: error["errInfo"] = result["errInfo"] res["writeErrors"] = [error] return res if operation == "insert": # GLE result for insert is always 0 in most MongoDB versions. res["n"] = len(command['documents']) elif operation == "update": if "upserted" in result: res["upserted"] = [{"index": 0, "_id": result["upserted"]}] # Versions of MongoDB before 2.6 don't return the _id for an # upsert if _id is not an ObjectId. elif result.get("updatedExisting") is False and affected == 1: # If _id is in both the update document *and* the query spec # the update document _id takes precedence. update = command['updates'][0] _id = update["u"].get("_id", update["q"].get("_id")) res["upserted"] = [{"index": 0, "_id": _id}] return res _OPTIONS = SON([ ('tailable', 2), ('oplogReplay', 8), ('noCursorTimeout', 16), ('awaitData', 32), ('allowPartialResults', 128)]) _MODIFIERS = SON([ ('$query', 'filter'), ('$orderby', 'sort'), ('$hint', 'hint'), ('$comment', 'comment'), ('$maxScan', 'maxScan'), ('$maxTimeMS', 'maxTimeMS'), ('$max', 'max'), ('$min', 'min'), ('$returnKey', 'returnKey'), ('$showRecordId', 'showRecordId'), ('$showDiskLoc', 'showRecordId'), # <= MongoDb 3.0 ('$snapshot', 'snapshot')]) def _gen_find_command(coll, spec, projection, skip, limit, batch_size, options, read_concern, collation=None): """Generate a find command document.""" cmd = SON([('find', coll)]) if '$query' in spec: cmd.update([(_MODIFIERS[key], val) if key in _MODIFIERS else (key, val) for key, val in spec.items()]) if '$explain' in cmd: cmd.pop('$explain') if '$readPreference' in cmd: cmd.pop('$readPreference') else: cmd['filter'] = spec if projection: cmd['projection'] = projection if skip: cmd['skip'] = skip if limit: cmd['limit'] = abs(limit) if limit < 0: cmd['singleBatch'] = True if batch_size: cmd['batchSize'] = batch_size if read_concern.level: cmd['readConcern'] = read_concern.document if collation: cmd['collation'] = collation if options: cmd.update([(opt, True) for opt, val in _OPTIONS.items() if options & val]) return cmd def _gen_get_more_command(cursor_id, coll, batch_size, max_await_time_ms): """Generate a getMore command document.""" cmd = SON([('getMore', cursor_id), ('collection', coll)]) if batch_size: cmd['batchSize'] = batch_size if max_await_time_ms is not None: cmd['maxTimeMS'] = max_await_time_ms return cmd class _Query(object): """A query operation.""" __slots__ = ('flags', 'db', 'coll', 'ntoskip', 'spec', 'fields', 'codec_options', 'read_preference', 'limit', 'batch_size', 'name', 'read_concern', 'collation', 'session', 'client') def __init__(self, flags, db, coll, ntoskip, spec, fields, codec_options, read_preference, limit, batch_size, read_concern, collation, session, client): self.flags = flags self.db = db self.coll = coll self.ntoskip = ntoskip self.spec = spec self.fields = fields self.codec_options = codec_options self.read_preference = read_preference self.read_concern = read_concern self.limit = limit self.batch_size = batch_size self.collation = collation self.session = session self.client = client self.name = 'find' def use_command(self, sock_info, exhaust): use_find_cmd = False if sock_info.max_wire_version >= 4: if not exhaust: use_find_cmd = True elif not self.read_concern.ok_for_legacy: raise ConfigurationError( 'read concern level of %s is not valid ' 'with a max wire version of %d.' % (self.read_concern.level, sock_info.max_wire_version)) if sock_info.max_wire_version < 5 and self.collation is not None: raise ConfigurationError( 'Specifying a collation is unsupported with a max wire ' 'version of %d.' % (sock_info.max_wire_version,)) sock_info.validate_session(self.client, self.session) return use_find_cmd def as_command(self, sock_info): """Return a find command document for this query. Should be called *after* get_message. """ explain = '$explain' in self.spec cmd = _gen_find_command( self.coll, self.spec, self.fields, self.ntoskip, self.limit, self.batch_size, self.flags, self.read_concern, self.collation) if explain: self.name = 'explain' cmd = SON([('explain', cmd)]) session = self.session if session: cmd['lsid'] = session._use_lsid() # Explain does not support readConcern. if (not explain and session.options.causal_consistency and session.operation_time is not None): cmd.setdefault( 'readConcern', {})[ 'afterClusterTime'] = session.operation_time sock_info.send_cluster_time(cmd, session, self.client) return cmd, self.db def get_message(self, set_slave_ok, sock_info, use_cmd=False): """Get a query message, possibly setting the slaveOk bit.""" if set_slave_ok: # Set the slaveOk bit. flags = self.flags | 4 else: flags = self.flags ns = _UJOIN % (self.db, self.coll) spec = self.spec if use_cmd: ns = _UJOIN % (self.db, "$cmd") spec = self.as_command(sock_info)[0] ntoreturn = -1 # All DB commands return 1 document else: # OP_QUERY treats ntoreturn of -1 and 1 the same, return # one document and close the cursor. We have to use 2 for # batch size if 1 is specified. ntoreturn = self.batch_size == 1 and 2 or self.batch_size if self.limit: if ntoreturn: ntoreturn = min(self.limit, ntoreturn) else: ntoreturn = self.limit if sock_info.is_mongos: spec = _maybe_add_read_preference(spec, self.read_preference) return query(flags, ns, self.ntoskip, ntoreturn, spec, None if use_cmd else self.fields, self.codec_options) class _GetMore(object): """A getmore operation.""" __slots__ = ('db', 'coll', 'ntoreturn', 'cursor_id', 'max_await_time_ms', 'codec_options', 'session', 'client') name = 'getMore' def __init__(self, db, coll, ntoreturn, cursor_id, codec_options, session, client, max_await_time_ms=None): self.db = db self.coll = coll self.ntoreturn = ntoreturn self.cursor_id = cursor_id self.codec_options = codec_options self.session = session self.client = client self.max_await_time_ms = max_await_time_ms def use_command(self, sock_info, exhaust): sock_info.validate_session(self.client, self.session) return sock_info.max_wire_version >= 4 and not exhaust def as_command(self, sock_info): """Return a getMore command document for this query.""" cmd = _gen_get_more_command(self.cursor_id, self.coll, self.ntoreturn, self.max_await_time_ms) if self.session: cmd['lsid'] = self.session._use_lsid() sock_info.send_cluster_time(cmd, self.session, self.client) return cmd, self.db def get_message(self, dummy0, sock_info, use_cmd=False): """Get a getmore message.""" ns = _UJOIN % (self.db, self.coll) if use_cmd: ns = _UJOIN % (self.db, "$cmd") spec = self.as_command(sock_info)[0] return query(0, ns, 0, -1, spec, None, self.codec_options) return get_more(ns, self.ntoreturn, self.cursor_id) class _RawBatchQuery(_Query): def use_command(self, socket_info, exhaust): # Compatibility checks. super(_RawBatchQuery, self).use_command(socket_info, exhaust) return False def get_message(self, set_slave_ok, sock_info, use_cmd=False): # Always pass False for use_cmd. return super(_RawBatchQuery, self).get_message( set_slave_ok, sock_info, False) class _RawBatchGetMore(_GetMore): def use_command(self, socket_info, exhaust): return False def get_message(self, set_slave_ok, sock_info, use_cmd=False): # Always pass False for use_cmd. return super(_RawBatchGetMore, self).get_message( set_slave_ok, sock_info, False) class _CursorAddress(tuple): """The server address (host, port) of a cursor, with namespace property.""" def __new__(cls, address, namespace): self = tuple.__new__(cls, address) self.__namespace = namespace return self @property def namespace(self): """The namespace this cursor.""" return self.__namespace def __hash__(self): # Two _CursorAddress instances with different namespaces # must not hash the same. return (self + (self.__namespace,)).__hash__() def __eq__(self, other): if isinstance(other, _CursorAddress): return (tuple(self) == tuple(other) and self.namespace == other.namespace) return NotImplemented def __ne__(self, other): return not self == other def __last_error(namespace, args): """Data to send to do a lastError. """ cmd = SON([("getlasterror", 1)]) cmd.update(args) splitns = namespace.split('.', 1) return query(0, splitns[0] + '.$cmd', 0, -1, cmd, None, DEFAULT_CODEC_OPTIONS) def __pack_message(operation, data): """Takes message data and adds a message header based on the operation. Returns the resultant message string. """ request_id = _randint() message = struct.pack(" ctx.max_bson_size) message_length += encoded_length if message_length < ctx.max_message_size and not too_large: data.write(encoded) to_send.append(doc) has_docs = True continue if has_docs: # We have enough data, send this message. try: request_id, msg = _insert_message(data.getvalue(), send_safe) ctx.legacy_write(request_id, msg, 0, send_safe, to_send) # Exception type could be OperationFailure or a subtype # (e.g. DuplicateKeyError) except OperationFailure as exc: # Like it says, continue on error... if continue_on_error: # Store exception details to re-raise after the final batch. last_error = exc # With unacknowledged writes just return at the first error. elif not safe: return # With acknowledged writes raise immediately. else: raise if too_large: _raise_document_too_large( "insert", encoded_length, ctx.max_bson_size) message_length = begin_loc + encoded_length data.seek(begin_loc) data.truncate() data.write(encoded) to_send = [doc] if not has_docs: raise InvalidOperation("cannot do an empty bulk insert") request_id, msg = _insert_message(data.getvalue(), safe) ctx.legacy_write(request_id, msg, 0, safe, to_send) # Re-raise any exception stored due to continue_on_error if last_error is not None: raise last_error if _use_c: _do_batched_insert = _cmessage._do_batched_insert def _do_batched_write_command(namespace, operation, command, docs, check_keys, opts, ctx): """Create the next batched insert, update, or delete command. """ max_bson_size = ctx.max_bson_size max_write_batch_size = ctx.max_write_batch_size # Max BSON object size + 16k - 2 bytes for ending NUL bytes. # Server guarantees there is enough room: SERVER-10643. max_cmd_size = max_bson_size + _COMMAND_OVERHEAD buf = StringIO() # Save space for message length and request id buf.write(_ZERO_64) # responseTo, opCode buf.write(b"\x00\x00\x00\x00\xd4\x07\x00\x00") # No options buf.write(_ZERO_32) # Namespace as C string buf.write(b(namespace)) buf.write(_ZERO_8) # Skip: 0, Limit: -1 buf.write(_SKIPLIM) # Where to write command document length command_start = buf.tell() buf.write(bson.BSON.encode(command)) # Start of payload buf.seek(-1, 2) # Work around some Jython weirdness. buf.truncate() try: buf.write(_OP_MAP[operation]) except KeyError: raise InvalidOperation('Unknown command') if operation in (_UPDATE, _DELETE): check_keys = False # Where to write list document length list_start = buf.tell() - 4 to_send = [] idx = 0 for doc in docs: # Encode the current operation key = b(str(idx)) value = bson.BSON.encode(doc, check_keys, opts) # Is there enough room to add this document? max_cmd_size accounts for # the two trailing null bytes. enough_data = (buf.tell() + len(key) + len(value)) >= max_cmd_size enough_documents = (idx >= max_write_batch_size) if enough_data or enough_documents: if not idx: write_op = "insert" if operation == _INSERT else None _raise_document_too_large( write_op, len(value), max_bson_size) break buf.write(_BSONOBJ) buf.write(key) buf.write(_ZERO_8) buf.write(value) to_send.append(doc) idx += 1 # Finalize the current OP_QUERY message. # Close list and command documents buf.write(_ZERO_16) # Write document lengths and request id length = buf.tell() buf.seek(list_start) buf.write(struct.pack('= 0") self.__batch_size = batch_size == 1 and 2 or batch_size return self def __send_message(self, operation): """Send a getmore message and handle the response. """ def kill(): self.__killed = True self.__end_session(True) client = self.__collection.database.client listeners = client._event_listeners publish = listeners.enabled_for_commands start = datetime.datetime.now() def duration(): return datetime.datetime.now() - start try: response = client._send_message_with_response( operation, address=self.__address) except AutoReconnect: # Don't try to send kill cursors on another socket # or to another server. It can cause a _pinValue # assertion on some server releases if we get here # due to a socket timeout. kill() raise rqst_id = response.request_id from_command = response.from_command reply = response.data try: docs = self._unpack_response(reply, self.__id, self.__collection.codec_options) if from_command: first = docs[0] client._receive_cluster_time(first, self.__session) helpers._check_command_response(first) except OperationFailure as exc: kill() if publish: listeners.publish_command_failure( duration(), exc.details, "getMore", rqst_id, self.__address) raise except NotMasterError as exc: # Don't send kill cursors to another server after a "not master" # error. It's completely pointless. kill() if publish: listeners.publish_command_failure( duration(), exc.details, "getMore", rqst_id, self.__address) client._reset_server_and_request_check(self.address) raise except Exception as exc: if publish: listeners.publish_command_failure( duration(), _convert_exception(exc), "getMore", rqst_id, self.__address) raise if from_command: cursor = docs[0]['cursor'] documents = cursor['nextBatch'] self.__id = cursor['id'] if publish: listeners.publish_command_success( duration(), docs[0], "getMore", rqst_id, self.__address) else: documents = docs self.__id = reply.cursor_id if publish: # Must publish in getMore command response format. res = {"cursor": {"id": self.__id, "ns": self.__collection.full_name, "nextBatch": documents}, "ok": 1} listeners.publish_command_success( duration(), res, "getMore", rqst_id, self.__address) if self.__id == 0: kill() self.__data = deque(documents) def _unpack_response(self, response, cursor_id, codec_options): return response.unpack_response(cursor_id, codec_options) def _refresh(self): """Refreshes the cursor with more data from the server. Returns the length of self.__data after refresh. Will exit early if self.__data is already non-empty. Raises OperationFailure when the cursor cannot be refreshed due to an error on the query. """ if len(self.__data) or self.__killed: return len(self.__data) if self.__id: # Get More dbname, collname = self.__ns.split('.', 1) self.__send_message( self._getmore_class(dbname, collname, self.__batch_size, self.__id, self.__collection.codec_options, self.__session, self.__collection.database.client, self.__max_await_time_ms)) else: # Cursor id is zero nothing else to return self.__killed = True self.__end_session(True) return len(self.__data) @property def alive(self): """Does this cursor have the potential to return more data? Even if :attr:`alive` is ``True``, :meth:`next` can raise :exc:`StopIteration`. Best to use a for loop:: for doc in collection.aggregate(pipeline): print(doc) .. note:: :attr:`alive` can be True while iterating a cursor from a failed server. In this case :attr:`alive` will return False after :meth:`next` fails to retrieve the next batch of results from the server. """ return bool(len(self.__data) or (not self.__killed)) @property def cursor_id(self): """Returns the id of the cursor.""" return self.__id @property def address(self): """The (host, port) of the server used, or None. .. versionadded:: 3.0 """ return self.__address @property def session(self): """The cursor's :class:`~pymongo.client_session.ClientSession`, or None. .. versionadded:: 3.6 """ if self.__explicit_session: return self.__session def __iter__(self): return self def next(self): """Advance the cursor.""" # Block until a document is returnable. while not len(self.__data) and not self.__killed: self._refresh() if len(self.__data): coll = self.__collection return coll.database._fix_outgoing(self.__data.popleft(), coll) else: raise StopIteration __next__ = next def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() class RawBatchCommandCursor(CommandCursor): _getmore_class = _RawBatchGetMore def __init__(self, collection, cursor_info, address, retrieved=0, batch_size=0, max_await_time_ms=None, session=None, explicit_session=False): """Create a new cursor / iterator over raw batches of BSON data. Should not be called directly by application developers - see :meth:`~pymongo.collection.Collection.aggregate_raw_batches` instead. .. mongodoc:: cursors """ assert not cursor_info.get('firstBatch') super(RawBatchCommandCursor, self).__init__( collection, cursor_info, address, retrieved, batch_size, max_await_time_ms, session, explicit_session) def _unpack_response(self, response, cursor_id, codec_options): return response.raw_response(cursor_id) def __getitem__(self, index): raise InvalidOperation("Cannot call __getitem__ on RawBatchCursor") pymongo-3.6.1/pymongo/operations.py0000644000076600000240000003230613245621354017671 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Operation class definitions.""" from pymongo.common import validate_boolean, validate_is_mapping, validate_list from pymongo.collation import validate_collation_or_none from pymongo.helpers import _gen_index_name, _index_document, _index_list class InsertOne(object): """Represents an insert_one operation.""" __slots__ = ("_doc",) def __init__(self, document): """Create an InsertOne instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `document`: The document to insert. If the document is missing an _id field one will be added. """ self._doc = document def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_insert(self._doc) def __repr__(self): return "InsertOne(%r)" % (self._doc,) def __eq__(self, other): if type(other) == type(self): return other._doc == self._doc return NotImplemented def __ne__(self, other): return not self == other class DeleteOne(object): """Represents a delete_one operation.""" __slots__ = ("_filter", "_collation") def __init__(self, filter, collation=None): """Create a DeleteOne instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the document to delete. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. .. versionchanged:: 3.5 Added the `collation` option. """ if filter is not None: validate_is_mapping("filter", filter) self._filter = filter self._collation = collation def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_delete(self._filter, 1, collation=self._collation) def __repr__(self): return "DeleteOne(%r, %r)" % (self._filter, self._collation) def __eq__(self, other): if type(other) == type(self): return ((other._filter, other._collation) == (self._filter, self._collation)) return NotImplemented def __ne__(self, other): return not self == other class DeleteMany(object): """Represents a delete_many operation.""" __slots__ = ("_filter", "_collation") def __init__(self, filter, collation=None): """Create a DeleteMany instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the documents to delete. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. .. versionchanged:: 3.5 Added the `collation` option. """ if filter is not None: validate_is_mapping("filter", filter) self._filter = filter self._collation = collation def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_delete(self._filter, 0, collation=self._collation) def __repr__(self): return "DeleteMany(%r, %r)" % (self._filter, self._collation) def __eq__(self, other): if type(other) == type(self): return ((other._filter, other._collation) == (self._filter, self._collation)) return NotImplemented def __ne__(self, other): return not self == other class ReplaceOne(object): """Represents a replace_one operation.""" __slots__ = ("_filter", "_doc", "_upsert", "_collation") def __init__(self, filter, replacement, upsert=False, collation=None): """Create a ReplaceOne instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the document to replace. - `replacement`: The new document. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. .. versionchanged:: 3.5 Added the `collation` option. """ if filter is not None: validate_is_mapping("filter", filter) if upsert is not None: validate_boolean("upsert", upsert) self._filter = filter self._doc = replacement self._upsert = upsert self._collation = collation def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_replace(self._filter, self._doc, self._upsert, collation=self._collation) def __eq__(self, other): if type(other) == type(self): return ( (other._filter, other._doc, other._upsert, other._collation) == (self._filter, self._doc, self._upsert, self._collation)) return NotImplemented def __ne__(self, other): return not self == other def __repr__(self): return "%s(%r, %r, %r, %r)" % ( self.__class__.__name__, self._filter, self._doc, self._upsert, self._collation) class _UpdateOp(object): """Private base class for update operations.""" __slots__ = ("_filter", "_doc", "_upsert", "_collation", "_array_filters") def __init__(self, filter, doc, upsert, collation, array_filters): if filter is not None: validate_is_mapping("filter", filter) if upsert is not None: validate_boolean("upsert", upsert) if array_filters is not None: validate_list("array_filters", array_filters) self._filter = filter self._doc = doc self._upsert = upsert self._collation = collation self._array_filters = array_filters def __eq__(self, other): if type(other) == type(self): return ( (other._filter, other._doc, other._upsert, other._collation, other._array_filters) == (self._filter, self._doc, self._upsert, self._collation, self._array_filters)) return NotImplemented def __ne__(self, other): return not self == other def __repr__(self): return "%s(%r, %r, %r, %r, %r)" % ( self.__class__.__name__, self._filter, self._doc, self._upsert, self._collation, self._array_filters) class UpdateOne(_UpdateOp): """Represents an update_one operation.""" __slots__ = () def __init__(self, filter, update, upsert=False, collation=None, array_filters=None): """Represents an update_one operation. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the document to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `array_filters` (optional): A list of filters specifying which array elements an update should apply. Requires MongoDB 3.6+. .. versionchanged:: 3.6 Added the `array_filters` option. .. versionchanged:: 3.5 Added the `collation` option. """ super(UpdateOne, self).__init__(filter, update, upsert, collation, array_filters) def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_update(self._filter, self._doc, False, self._upsert, collation=self._collation, array_filters=self._array_filters) class UpdateMany(_UpdateOp): """Represents an update_many operation.""" __slots__ = () def __init__(self, filter, update, upsert=False, collation=None, array_filters=None): """Create an UpdateMany instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the documents to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `array_filters` (optional): A list of filters specifying which array elements an update should apply. Requires MongoDB 3.6+. .. versionchanged:: 3.6 Added the `array_filters` option. .. versionchanged:: 3.5 Added the `collation` option. """ super(UpdateMany, self).__init__(filter, update, upsert, collation, array_filters) def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_update(self._filter, self._doc, True, self._upsert, collation=self._collation, array_filters=self._array_filters) class IndexModel(object): """Represents an index to create.""" __slots__ = ("__document",) def __init__(self, keys, **kwargs): """Create an Index instance. For use with :meth:`~pymongo.collection.Collection.create_indexes`. Takes either a single key or a list of (key, direction) pairs. The key(s) must be an instance of :class:`basestring` (:class:`str` in python 3), and the direction(s) must be one of (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`, :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`, :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`, :data:`~pymongo.TEXT`). Valid options include, but are not limited to: - `name`: custom name to use for this index - if none is given, a name will be generated. - `unique`: if ``True`` creates a uniqueness constraint on the index. - `background`: if ``True`` this index should be created in the background. - `sparse`: if ``True``, omit from the index any documents that lack the indexed field. - `bucketSize`: for use with geoHaystack indexes. Number of documents to group together within a certain proximity to a given longitude and latitude. - `min`: minimum value for keys in a :data:`~pymongo.GEO2D` index. - `max`: maximum value for keys in a :data:`~pymongo.GEO2D` index. - `expireAfterSeconds`: Used to create an expiring (TTL) collection. MongoDB will automatically delete documents from this collection after seconds. The indexed field must be a UTC datetime or the data will not expire. - `partialFilterExpression`: A document that specifies a filter for a partial index. - `collation`: An instance of :class:`~pymongo.collation.Collation` that specifies the collation to use in MongoDB >= 3.4. See the MongoDB documentation for a full list of supported options by server version. .. note:: `partialFilterExpression` requires server version **>= 3.2** :Parameters: - `keys`: a single key or a list of (key, direction) pairs specifying the index to create - `**kwargs` (optional): any additional index creation options (see the above list) should be passed as keyword arguments .. versionchanged:: 3.2 Added partialFilterExpression to support partial indexes. """ keys = _index_list(keys) if "name" not in kwargs: kwargs["name"] = _gen_index_name(keys) kwargs["key"] = _index_document(keys) collation = validate_collation_or_none(kwargs.pop('collation', None)) self.__document = kwargs if collation is not None: self.__document['collation'] = collation @property def document(self): """An index document suitable for passing to the createIndexes command. """ return self.__document pymongo-3.6.1/pymongo/write_concern.py0000644000076600000240000001063213245621354020345 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for working with write concerns.""" from bson.py3compat import integer_types, string_type from pymongo.errors import ConfigurationError class WriteConcern(object): """WriteConcern :Parameters: - `w`: (integer or string) Used with replication, write operations will block until they have been replicated to the specified number or tagged set of servers. `w=` always includes the replica set primary (e.g. w=3 means write to the primary and wait until replicated to **two** secondaries). **w=0 disables acknowledgement of write operations and can not be used with other write concern options.** - `wtimeout`: (integer) Used in conjunction with `w`. Specify a value in milliseconds to control how long to wait for write propagation to complete. If replication does not complete in the given timeframe, a timeout exception is raised. - `j`: If ``True`` block until write operations have been committed to the journal. Cannot be used in combination with `fsync`. Prior to MongoDB 2.6 this option was ignored if the server was running without journaling. Starting with MongoDB 2.6 write operations will fail with an exception if this option is used when the server is running without journaling. - `fsync`: If ``True`` and the server is running without journaling, blocks until the server has synced all data files to disk. If the server is running with journaling, this acts the same as the `j` option, blocking until write operations have been committed to the journal. Cannot be used in combination with `j`. """ __slots__ = ("__document", "__acknowledged") def __init__(self, w=None, wtimeout=None, j=None, fsync=None): self.__document = {} self.__acknowledged = True if wtimeout is not None: if not isinstance(wtimeout, integer_types): raise TypeError("wtimeout must be an integer") self.__document["wtimeout"] = wtimeout if j is not None: if not isinstance(j, bool): raise TypeError("j must be True or False") self.__document["j"] = j if fsync is not None: if not isinstance(fsync, bool): raise TypeError("fsync must be True or False") if j and fsync: raise ConfigurationError("Can't set both j " "and fsync at the same time") self.__document["fsync"] = fsync if self.__document and w == 0: raise ConfigurationError("Can not use w value " "of 0 with other options") if w is not None: if isinstance(w, integer_types): self.__acknowledged = w > 0 elif not isinstance(w, string_type): raise TypeError("w must be an integer or string") self.__document["w"] = w @property def document(self): """The document representation of this write concern. .. note:: :class:`WriteConcern` is immutable. Mutating the value of :attr:`document` does not mutate this :class:`WriteConcern`. """ return self.__document.copy() @property def acknowledged(self): """If ``True`` write operations will wait for acknowledgement before returning. """ return self.__acknowledged def __repr__(self): return ("WriteConcern(%s)" % ( ", ".join("%s=%s" % kvt for kvt in self.document.items()),)) def __eq__(self, other): return self.document == other.document def __ne__(self, other): return self.document != other.document def __bool__(self): return bool(self.document) pymongo-3.6.1/pymongo/uri_parser.py0000644000076600000240000003655013245621354017666 0ustar shanestaff00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Tools to parse and validate a MongoDB URI.""" import re import warnings try: from dns import resolver _HAVE_DNSPYTHON = True except ImportError: _HAVE_DNSPYTHON = False from bson.py3compat import PY3, string_type if PY3: from urllib.parse import unquote_plus else: from urllib import unquote_plus from pymongo.common import get_validated_options from pymongo.errors import ConfigurationError, InvalidURI SCHEME = 'mongodb://' SCHEME_LEN = len(SCHEME) SRV_SCHEME = 'mongodb+srv://' SRV_SCHEME_LEN = len(SRV_SCHEME) DEFAULT_PORT = 27017 def _partition(entity, sep): """Python2.4 doesn't have a partition method so we provide our own that mimics str.partition from later releases. Split the string at the first occurrence of sep, and return a 3-tuple containing the part before the separator, the separator itself, and the part after the separator. If the separator is not found, return a 3-tuple containing the string itself, followed by two empty strings. """ parts = entity.split(sep, 1) if len(parts) == 2: return parts[0], sep, parts[1] else: return entity, '', '' def _rpartition(entity, sep): """Python2.4 doesn't have an rpartition method so we provide our own that mimics str.rpartition from later releases. Split the string at the last occurrence of sep, and return a 3-tuple containing the part before the separator, the separator itself, and the part after the separator. If the separator is not found, return a 3-tuple containing two empty strings, followed by the string itself. """ idx = entity.rfind(sep) if idx == -1: return '', '', entity return entity[:idx], sep, entity[idx + 1:] def parse_userinfo(userinfo): """Validates the format of user information in a MongoDB URI. Reserved characters like ':', '/', '+' and '@' must be escaped following RFC 3986. Returns a 2-tuple containing the unescaped username followed by the unescaped password. :Paramaters: - `userinfo`: A string of the form : .. versionchanged:: 2.2 Now uses `urllib.unquote_plus` so `+` characters must be escaped. """ if '@' in userinfo or userinfo.count(':') > 1: if PY3: quote_fn = "urllib.parse.quote_plus" else: quote_fn = "urllib.quote_plus" raise InvalidURI("Username and password must be escaped according to " "RFC 3986, use %s()." % quote_fn) user, _, passwd = _partition(userinfo, ":") # No password is expected with GSSAPI authentication. if not user: raise InvalidURI("The empty string is not valid username.") return unquote_plus(user), unquote_plus(passwd) def parse_ipv6_literal_host(entity, default_port): """Validates an IPv6 literal host:port string. Returns a 2-tuple of IPv6 literal followed by port where port is default_port if it wasn't specified in entity. :Parameters: - `entity`: A string that represents an IPv6 literal enclosed in braces (e.g. '[::1]' or '[::1]:27017'). - `default_port`: The port number to use when one wasn't specified in entity. """ if entity.find(']') == -1: raise ValueError("an IPv6 address literal must be " "enclosed in '[' and ']' according " "to RFC 2732.") i = entity.find(']:') if i == -1: return entity[1:-1], default_port return entity[1: i], entity[i + 2:] def parse_host(entity, default_port=DEFAULT_PORT): """Validates a host string Returns a 2-tuple of host followed by port where port is default_port if it wasn't specified in the string. :Parameters: - `entity`: A host or host:port string where host could be a hostname or IP address. - `default_port`: The port number to use when one wasn't specified in entity. """ host = entity port = default_port if entity[0] == '[': host, port = parse_ipv6_literal_host(entity, default_port) elif entity.endswith(".sock"): return entity, default_port elif entity.find(':') != -1: if entity.count(':') > 1: raise ValueError("Reserved characters such as ':' must be " "escaped according RFC 2396. An IPv6 " "address literal must be enclosed in '[' " "and ']' according to RFC 2732.") host, port = host.split(':', 1) if isinstance(port, string_type): if not port.isdigit() or int(port) > 65535 or int(port) <= 0: raise ValueError("Port must be an integer between 0 and 65535: %s" % (port,)) port = int(port) # Normalize hostname to lowercase, since DNS is case-insensitive: # http://tools.ietf.org/html/rfc4343 # This prevents useless rediscovery if "foo.com" is in the seed list but # "FOO.com" is in the ismaster response. return host.lower(), port def validate_options(opts, warn=False): """Validates and normalizes options passed in a MongoDB URI. Returns a new dictionary of validated and normalized options. If warn is False then errors will be thrown for invalid options, otherwise they will be ignored and a warning will be issued. :Parameters: - `opts`: A dict of MongoDB URI options. - `warn` (optional): If ``True`` then warnigns will be logged and invalid options will be ignored. Otherwise invalid options will cause errors. """ return get_validated_options(opts, warn) def _parse_options(opts, delim): """Helper method for split_options which creates the options dict. Also handles the creation of a list for the URI tag_sets/ readpreferencetags portion.""" options = {} for opt in opts.split(delim): key, val = opt.split("=") if key.lower() == 'readpreferencetags': options.setdefault('readpreferencetags', []).append(val) else: # str(option) to ensure that a unicode URI results in plain 'str' # option names. 'normalized' is then suitable to be passed as # kwargs in all Python versions. if str(key) in options: warnings.warn("Duplicate URI option %s" % (str(key),)) options[str(key)] = unquote_plus(val) # Special case for deprecated options if "wtimeout" in options: if "wtimeoutMS" in options: options.pop("wtimeout") warnings.warn("Option wtimeout is deprecated, use 'wtimeoutMS'" " instead") return options def split_options(opts, validate=True, warn=False): """Takes the options portion of a MongoDB URI, validates each option and returns the options in a dictionary. :Parameters: - `opt`: A string representing MongoDB URI options. - `validate`: If ``True`` (the default), validate and normalize all options. """ and_idx = opts.find("&") semi_idx = opts.find(";") try: if and_idx >= 0 and semi_idx >= 0: raise InvalidURI("Can not mix '&' and ';' for option separators.") elif and_idx >= 0: options = _parse_options(opts, "&") elif semi_idx >= 0: options = _parse_options(opts, ";") elif opts.find("=") != -1: options = _parse_options(opts, None) else: raise ValueError except ValueError: raise InvalidURI("MongoDB URI options are key=value pairs.") if validate: return validate_options(options, warn) return options def split_hosts(hosts, default_port=DEFAULT_PORT): """Takes a string of the form host1[:port],host2[:port]... and splits it into (host, port) tuples. If [:port] isn't present the default_port is used. Returns a set of 2-tuples containing the host name (or IP) followed by port number. :Parameters: - `hosts`: A string of the form host1[:port],host2[:port],... - `default_port`: The port number to use when one wasn't specified for a host. """ nodes = [] for entity in hosts.split(','): if not entity: raise ConfigurationError("Empty host " "(or extra comma in host list).") port = default_port # Unix socket entities don't have ports if entity.endswith('.sock'): port = None nodes.append(parse_host(entity, port)) return nodes # Prohibited characters in database name. DB names also can't have ".", but for # backward-compat we allow "db.collection" in URI. _BAD_DB_CHARS = re.compile('[' + re.escape(r'/ "$') + ']') if PY3: # dnspython can return bytes or str from various parts # of its API depending on version. We always want str. def maybe_decode(text): if isinstance(text, bytes): return text.decode() return text else: def maybe_decode(text): return text _ALLOWED_TXT_OPTS = frozenset( ['authsource', 'authSource', 'replicaset', 'replicaSet']) def _get_dns_srv_hosts(hostname): try: results = resolver.query('_mongodb._tcp.' + hostname, 'SRV') except Exception as exc: raise ConfigurationError(str(exc)) return [(maybe_decode(res.target.to_text(omit_final_dot=True)), res.port) for res in results] def _get_dns_txt_options(hostname): try: results = resolver.query(hostname, 'TXT') except resolver.NoAnswer: # No TXT records return None except Exception as exc: raise ConfigurationError(str(exc)) if len(results) > 1: raise ConfigurationError('Only one TXT record is supported') return ( b'&'.join([b''.join(res.strings) for res in results])).decode('utf-8') def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False): """Parse and validate a MongoDB URI. Returns a dict of the form:: { 'nodelist': , 'username': or None, 'password': or None, 'database': or None, 'collection': or None, 'options': } If the URI scheme is "mongodb+srv://" DNS SRV and TXT lookups will be done to build nodelist and options. :Parameters: - `uri`: The MongoDB URI to parse. - `default_port`: The port number to use when one wasn't specified for a host in the URI. - `validate`: If ``True`` (the default), validate and normalize all options. - `warn` (optional): When validating, if ``True`` then will warn the user then ignore any invalid options or values. If ``False``, validation will error when options are unsupported or values are invalid. .. versionchanged:: 3.6 Added support for mongodb+srv:// URIs .. versionchanged:: 3.5 Return the original value of the ``readPreference`` MongoDB URI option instead of the validated read preference mode. .. versionchanged:: 3.1 ``warn`` added so invalid options can be ignored. """ if uri.startswith(SCHEME): is_srv = False scheme_free = uri[SCHEME_LEN:] elif uri.startswith(SRV_SCHEME): if not _HAVE_DNSPYTHON: raise ConfigurationError('The "dnspython" module must be ' 'installed to use mongodb+srv:// URIs') is_srv = True scheme_free = uri[SRV_SCHEME_LEN:] else: raise InvalidURI("Invalid URI scheme: URI must " "begin with '%s' or '%s'" % (SCHEME, SRV_SCHEME)) if not scheme_free: raise InvalidURI("Must provide at least one hostname or IP.") user = None passwd = None dbase = None collection = None options = {} host_part, _, path_part = _partition(scheme_free, '/') if not host_part: host_part = path_part path_part = "" if not path_part and '?' in host_part: raise InvalidURI("A '/' is required between " "the host list and any options.") if '@' in host_part: userinfo, _, hosts = _rpartition(host_part, '@') user, passwd = parse_userinfo(userinfo) else: hosts = host_part if '/' in hosts: raise InvalidURI("Any '/' in a unix domain socket must be" " percent-encoded: %s" % host_part) hosts = unquote_plus(hosts) if is_srv: nodes = split_hosts(hosts, default_port=None) if len(nodes) != 1: raise InvalidURI( "%s URIs must include one, " "and only one, hostname" % (SRV_SCHEME,)) fqdn, port = nodes[0] if port is not None: raise InvalidURI( "%s URIs must not include a port number" % (SRV_SCHEME,)) nodes = _get_dns_srv_hosts(fqdn) try: plist = fqdn.split(".")[1:] except Exception: raise ConfigurationError("Invalid URI host") slen = len(plist) if slen < 2: raise ConfigurationError("Invalid URI host") for node in nodes: try: nlist = node[0].split(".")[1:][-slen:] except Exception: raise ConfigurationError("Invalid SRV host") if plist != nlist: raise ConfigurationError("Invalid SRV host") dns_options = _get_dns_txt_options(fqdn) if dns_options: options = split_options(dns_options, validate, warn) if set(options) - _ALLOWED_TXT_OPTS: raise ConfigurationError( "Only authSource and replicaSet are supported from DNS") options["ssl"] = True if validate else 'true' else: nodes = split_hosts(hosts, default_port=default_port) if path_part: if path_part[0] == '?': opts = unquote_plus(path_part[1:]) else: dbase, _, opts = map(unquote_plus, _partition(path_part, '?')) if '.' in dbase: dbase, collection = dbase.split('.', 1) if _BAD_DB_CHARS.search(dbase): raise InvalidURI('Bad database name "%s"' % dbase) if opts: options.update(split_options(opts, validate, warn)) if dbase is not None: dbase = unquote_plus(dbase) if collection is not None: collection = unquote_plus(collection) return { 'nodelist': nodes, 'username': user, 'password': passwd, 'database': dbase, 'collection': collection, 'options': options } if __name__ == '__main__': import pprint import sys try: pprint.pprint(parse_uri(sys.argv[1])) except InvalidURI as exc: print(exc) sys.exit(0) pymongo-3.6.1/pymongo/common.py0000644000076600000240000005271713245621354017006 0ustar shanestaff00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Functions and classes common to multiple pymongo modules.""" import collections import datetime import warnings from bson import SON from bson.binary import (STANDARD, PYTHON_LEGACY, JAVA_LEGACY, CSHARP_LEGACY) from bson.codec_options import CodecOptions from bson.py3compat import abc, integer_types, iteritems, string_type from bson.raw_bson import RawBSONDocument from pymongo.auth import MECHANISMS from pymongo.errors import ConfigurationError from pymongo.monitoring import _validate_event_listeners from pymongo.read_concern import ReadConcern from pymongo.read_preferences import _MONGOS_MODES, _ServerMode from pymongo.ssl_support import validate_cert_reqs from pymongo.write_concern import WriteConcern try: from collections import OrderedDict ORDERED_TYPES = (SON, OrderedDict) except ImportError: ORDERED_TYPES = (SON,) # Defaults until we connect to a server and get updated limits. MAX_BSON_SIZE = 16 * (1024 ** 2) MAX_MESSAGE_SIZE = 2 * MAX_BSON_SIZE MIN_WIRE_VERSION = 0 MAX_WIRE_VERSION = 0 MAX_WRITE_BATCH_SIZE = 1000 # What this version of PyMongo supports. MIN_SUPPORTED_SERVER_VERSION = "2.6" MIN_SUPPORTED_WIRE_VERSION = 2 MAX_SUPPORTED_WIRE_VERSION = 5 # Frequency to call ismaster on servers, in seconds. HEARTBEAT_FREQUENCY = 10 # Frequency to process kill-cursors, in seconds. See MongoClient.close_cursor. KILL_CURSOR_FREQUENCY = 1 # Frequency to process events queue, in seconds. EVENTS_QUEUE_FREQUENCY = 1 # How long to wait, in seconds, for a suitable server to be found before # aborting an operation. For example, if the client attempts an insert # during a replica set election, SERVER_SELECTION_TIMEOUT governs the # longest it is willing to wait for a new primary to be found. SERVER_SELECTION_TIMEOUT = 30 # Spec requires at least 500ms between ismaster calls. MIN_HEARTBEAT_INTERVAL = 0.5 # Default connectTimeout in seconds. CONNECT_TIMEOUT = 20.0 # Default value for maxPoolSize. MAX_POOL_SIZE = 100 # Default value for minPoolSize. MIN_POOL_SIZE = 0 # Default value for maxIdleTimeMS. MAX_IDLE_TIME_MS = None # Default value for localThresholdMS. LOCAL_THRESHOLD_MS = 15 # Default value for retryWrites. RETRY_WRITES = False # mongod/s 2.6 and above return code 59 when a command doesn't exist. COMMAND_NOT_FOUND_CODES = (59,) # Error codes to ignore if GridFS calls createIndex on a secondary UNAUTHORIZED_CODES = (13, 16547, 16548) # Maximum number of sessions to send in a single endSessions command. # From the driver sessions spec. _MAX_END_SESSIONS = 10000 def partition_node(node): """Split a host:port string into (host, int(port)) pair.""" host = node port = 27017 idx = node.rfind(':') if idx != -1: host, port = node[:idx], int(node[idx + 1:]) if host.startswith('['): host = host[1:-1] return host, port def clean_node(node): """Split and normalize a node name from an ismaster response.""" host, port = partition_node(node) # Normalize hostname to lowercase, since DNS is case-insensitive: # http://tools.ietf.org/html/rfc4343 # This prevents useless rediscovery if "foo.com" is in the seed list but # "FOO.com" is in the ismaster response. return host.lower(), port def raise_config_error(key, dummy): """Raise ConfigurationError with the given key name.""" raise ConfigurationError("Unknown option %s" % (key,)) # Mapping of URI uuid representation options to valid subtypes. _UUID_REPRESENTATIONS = { 'standard': STANDARD, 'pythonLegacy': PYTHON_LEGACY, 'javaLegacy': JAVA_LEGACY, 'csharpLegacy': CSHARP_LEGACY } def validate_boolean(option, value): """Validates that 'value' is True or False.""" if isinstance(value, bool): return value raise TypeError("%s must be True or False" % (option,)) def validate_boolean_or_string(option, value): """Validates that value is True, False, 'true', or 'false'.""" if isinstance(value, string_type): if value not in ('true', 'false'): raise ValueError("The value of %s must be " "'true' or 'false'" % (option,)) return value == 'true' return validate_boolean(option, value) def validate_integer(option, value): """Validates that 'value' is an integer (or basestring representation). """ if isinstance(value, integer_types): return value elif isinstance(value, string_type): if not value.isdigit(): raise ValueError("The value of %s must be " "an integer" % (option,)) return int(value) raise TypeError("Wrong type for %s, value must be an integer" % (option,)) def validate_positive_integer(option, value): """Validate that 'value' is a positive integer, which does not include 0. """ val = validate_integer(option, value) if val <= 0: raise ValueError("The value of %s must be " "a positive integer" % (option,)) return val def validate_non_negative_integer(option, value): """Validate that 'value' is a positive integer or 0. """ val = validate_integer(option, value) if val < 0: raise ValueError("The value of %s must be " "a non negative integer" % (option,)) return val def validate_readable(option, value): """Validates that 'value' is file-like and readable. """ if value is None: return value # First make sure its a string py3.3 open(True, 'r') succeeds # Used in ssl cert checking due to poor ssl module error reporting value = validate_string(option, value) open(value, 'r').close() return value def validate_positive_integer_or_none(option, value): """Validate that 'value' is a positive integer or None. """ if value is None: return value return validate_positive_integer(option, value) def validate_non_negative_integer_or_none(option, value): """Validate that 'value' is a positive integer or 0 or None. """ if value is None: return value return validate_non_negative_integer(option, value) def validate_string(option, value): """Validates that 'value' is an instance of `basestring` for Python 2 or `str` for Python 3. """ if isinstance(value, string_type): return value raise TypeError("Wrong type for %s, value must be " "an instance of %s" % (option, string_type.__name__)) def validate_string_or_none(option, value): """Validates that 'value' is an instance of `basestring` or `None`. """ if value is None: return value return validate_string(option, value) def validate_int_or_basestring(option, value): """Validates that 'value' is an integer or string. """ if isinstance(value, integer_types): return value elif isinstance(value, string_type): if value.isdigit(): return int(value) return value raise TypeError("Wrong type for %s, value must be an " "integer or a string" % (option,)) def validate_positive_float(option, value): """Validates that 'value' is a float, or can be converted to one, and is positive. """ errmsg = "%s must be an integer or float" % (option,) try: value = float(value) except ValueError: raise ValueError(errmsg) except TypeError: raise TypeError(errmsg) # float('inf') doesn't work in 2.4 or 2.5 on Windows, so just cap floats at # one billion - this is a reasonable approximation for infinity if not 0 < value < 1e9: raise ValueError("%s must be greater than 0 and " "less than one billion" % (option,)) return value def validate_positive_float_or_zero(option, value): """Validates that 'value' is 0 or a positive float, or can be converted to 0 or a positive float. """ if value == 0 or value == "0": return 0 return validate_positive_float(option, value) def validate_timeout_or_none(option, value): """Validates a timeout specified in milliseconds returning a value in floating point seconds. """ if value is None: return value return validate_positive_float(option, value) / 1000.0 def validate_timeout_or_zero(option, value): """Validates a timeout specified in milliseconds returning a value in floating point seconds for the case where None is an error and 0 is valid. Setting the timeout to nothing in the URI string is a config error. """ if value is None: raise ConfigurationError("%s cannot be None" % (option, )) if value == 0 or value == "0": return 0 return validate_positive_float(option, value) / 1000.0 def validate_max_staleness(option, value): """Validates maxStalenessSeconds according to the Max Staleness Spec.""" if value == -1 or value == "-1": # Default: No maximum staleness. return -1 return validate_positive_integer(option, value) def validate_read_preference(dummy, value): """Validate a read preference. """ if not isinstance(value, _ServerMode): raise TypeError("%r is not a read preference." % (value,)) return value def validate_read_preference_mode(dummy, value): """Validate read preference mode for a MongoReplicaSetClient. .. versionchanged:: 3.5 Returns the original ``value`` instead of the validated read preference mode. """ if value not in _MONGOS_MODES: raise ValueError("%s is not a valid read preference" % (value,)) return value def validate_auth_mechanism(option, value): """Validate the authMechanism URI option. """ # CRAM-MD5 is for server testing only. Undocumented, # unsupported, may be removed at any time. You have # been warned. if value not in MECHANISMS and value != 'CRAM-MD5': raise ValueError("%s must be in %s" % (option, tuple(MECHANISMS))) return value def validate_uuid_representation(dummy, value): """Validate the uuid representation option selected in the URI. """ try: return _UUID_REPRESENTATIONS[value] except KeyError: raise ValueError("%s is an invalid UUID representation. " "Must be one of " "%s" % (value, tuple(_UUID_REPRESENTATIONS))) def validate_read_preference_tags(name, value): """Parse readPreferenceTags if passed as a client kwarg. """ if not isinstance(value, list): value = [value] tag_sets = [] for tag_set in value: if tag_set == '': tag_sets.append({}) continue try: tag_sets.append(dict([tag.split(":") for tag in tag_set.split(",")])) except Exception: raise ValueError("%r not a valid " "value for %s" % (tag_set, name)) return tag_sets _MECHANISM_PROPS = frozenset(['SERVICE_NAME', 'CANONICALIZE_HOST_NAME', 'SERVICE_REALM']) def validate_auth_mechanism_properties(option, value): """Validate authMechanismProperties.""" value = validate_string(option, value) props = {} for opt in value.split(','): try: key, val = opt.split(':') except ValueError: raise ValueError("auth mechanism properties must be " "key:value pairs like SERVICE_NAME:" "mongodb, not %s." % (opt,)) if key not in _MECHANISM_PROPS: raise ValueError("%s is not a supported auth " "mechanism property. Must be one of " "%s." % (key, tuple(_MECHANISM_PROPS))) if key == 'CANONICALIZE_HOST_NAME': props[key] = validate_boolean_or_string(key, val) else: props[key] = val return props def validate_document_class(option, value): """Validate the document_class option.""" if not issubclass(value, (abc.MutableMapping, RawBSONDocument)): raise TypeError("%s must be dict, bson.son.SON, " "bson.raw_bson.RawBSONDocument, or a " "sublass of collections.MutableMapping" % (option,)) return value def validate_list(option, value): """Validates that 'value' is a list.""" if not isinstance(value, list): raise TypeError("%s must be a list" % (option,)) return value def validate_list_or_none(option, value): """Validates that 'value' is a list or None.""" if value is None: return value return validate_list(option, value) def validate_is_mapping(option, value): """Validate the type of method arguments that expect a document.""" if not isinstance(value, abc.Mapping): raise TypeError("%s must be an instance of dict, bson.son.SON, or " "other type that inherits from " "collections.Mapping" % (option,)) def validate_is_document_type(option, value): """Validate the type of method arguments that expect a MongoDB document.""" if not isinstance(value, (abc.MutableMapping, RawBSONDocument)): raise TypeError("%s must be an instance of dict, bson.son.SON, " "bson.raw_bson.RawBSONDocument, or " "a type that inherits from " "collections.MutableMapping" % (option,)) def validate_appname_or_none(option, value): """Validate the appname option.""" if value is None: return value validate_string(option, value) # We need length in bytes, so encode utf8 first. if len(value.encode('utf-8')) > 128: raise ValueError("%s must be <= 128 bytes" % (option,)) return value def validate_ok_for_replace(replacement): """Validate a replacement document.""" validate_is_mapping("replacement", replacement) # Replacement can be {} if replacement and not isinstance(replacement, RawBSONDocument): first = next(iter(replacement)) if first.startswith('$'): raise ValueError('replacement can not include $ operators') def validate_ok_for_update(update): """Validate an update document.""" validate_is_mapping("update", update) # Update can not be {} if not update: raise ValueError('update only works with $ operators') first = next(iter(update)) if not first.startswith('$'): raise ValueError('update only works with $ operators') _UNICODE_DECODE_ERROR_HANDLERS = frozenset(['strict', 'replace', 'ignore']) def validate_unicode_decode_error_handler(dummy, value): """Validate the Unicode decode error handler option of CodecOptions. """ if value not in _UNICODE_DECODE_ERROR_HANDLERS: raise ValueError("%s is an invalid Unicode decode error handler. " "Must be one of " "%s" % (value, tuple(_UNICODE_DECODE_ERROR_HANDLERS))) return value def validate_tzinfo(dummy, value): """Validate the tzinfo option """ if value is not None and not isinstance(value, datetime.tzinfo): raise TypeError("%s must be an instance of datetime.tzinfo" % value) return value # journal is an alias for j, # wtimeoutms is an alias for wtimeout, URI_VALIDATORS = { 'replicaset': validate_string_or_none, 'w': validate_int_or_basestring, 'wtimeout': validate_integer, 'wtimeoutms': validate_integer, 'fsync': validate_boolean_or_string, 'j': validate_boolean_or_string, 'journal': validate_boolean_or_string, 'maxpoolsize': validate_positive_integer_or_none, 'socketkeepalive': validate_boolean_or_string, 'waitqueuemultiple': validate_non_negative_integer_or_none, 'ssl': validate_boolean_or_string, 'ssl_keyfile': validate_readable, 'ssl_certfile': validate_readable, 'ssl_pem_passphrase': validate_string_or_none, 'ssl_cert_reqs': validate_cert_reqs, 'ssl_ca_certs': validate_readable, 'ssl_match_hostname': validate_boolean_or_string, 'ssl_crlfile': validate_readable, 'readconcernlevel': validate_string_or_none, 'readpreference': validate_read_preference_mode, 'readpreferencetags': validate_read_preference_tags, 'localthresholdms': validate_positive_float_or_zero, 'authmechanism': validate_auth_mechanism, 'authsource': validate_string, 'authmechanismproperties': validate_auth_mechanism_properties, 'tz_aware': validate_boolean_or_string, 'uuidrepresentation': validate_uuid_representation, 'connect': validate_boolean_or_string, 'minpoolsize': validate_non_negative_integer, 'appname': validate_appname_or_none, 'unicode_decode_error_handler': validate_unicode_decode_error_handler, 'retrywrites': validate_boolean_or_string, } TIMEOUT_VALIDATORS = { 'connecttimeoutms': validate_timeout_or_none, 'sockettimeoutms': validate_timeout_or_none, 'waitqueuetimeoutms': validate_timeout_or_none, 'serverselectiontimeoutms': validate_timeout_or_zero, 'heartbeatfrequencyms': validate_timeout_or_none, 'maxidletimems': validate_timeout_or_none, 'maxstalenessseconds': validate_max_staleness, } KW_VALIDATORS = { 'document_class': validate_document_class, 'read_preference': validate_read_preference, 'event_listeners': _validate_event_listeners, 'tzinfo': validate_tzinfo, 'username': validate_string_or_none, 'password': validate_string_or_none, } URI_VALIDATORS.update(TIMEOUT_VALIDATORS) VALIDATORS = URI_VALIDATORS.copy() VALIDATORS.update(KW_VALIDATORS) _AUTH_OPTIONS = frozenset(['authmechanismproperties']) def validate_auth_option(option, value): """Validate optional authentication parameters. """ lower, value = validate(option, value) if lower not in _AUTH_OPTIONS: raise ConfigurationError('Unknown ' 'authentication option: %s' % (option,)) return lower, value def validate(option, value): """Generic validation function. """ lower = option.lower() validator = VALIDATORS.get(lower, raise_config_error) value = validator(option, value) return lower, value def get_validated_options(options, warn=True): """Validate each entry in options and raise a warning if it is not valid. Returns a copy of options with invalid entries removed """ validated_options = {} for opt, value in iteritems(options): lower = opt.lower() try: validator = URI_VALIDATORS.get(lower, raise_config_error) value = validator(opt, value) except (ValueError, ConfigurationError) as exc: if warn: warnings.warn(str(exc)) else: raise else: validated_options[lower] = value return validated_options WRITE_CONCERN_OPTIONS = frozenset([ 'w', 'wtimeout', 'wtimeoutms', 'fsync', 'j', 'journal' ]) class BaseObject(object): """A base class that provides attributes and methods common to multiple pymongo classes. SHOULD NOT BE USED BY DEVELOPERS EXTERNAL TO MONGODB. """ def __init__(self, codec_options, read_preference, write_concern, read_concern): if not isinstance(codec_options, CodecOptions): raise TypeError("codec_options must be an instance of " "bson.codec_options.CodecOptions") self.__codec_options = codec_options if not isinstance(read_preference, _ServerMode): raise TypeError("%r is not valid for read_preference. See " "pymongo.read_preferences for valid " "options." % (read_preference,)) self.__read_preference = read_preference if not isinstance(write_concern, WriteConcern): raise TypeError("write_concern must be an instance of " "pymongo.write_concern.WriteConcern") self.__write_concern = write_concern if not isinstance(read_concern, ReadConcern): raise TypeError("read_concern must be an instance of " "pymongo.read_concern.ReadConcern") self.__read_concern = read_concern @property def codec_options(self): """Read only access to the :class:`~bson.codec_options.CodecOptions` of this instance. """ return self.__codec_options @property def write_concern(self): """Read only access to the :class:`~pymongo.write_concern.WriteConcern` of this instance. .. versionchanged:: 3.0 The :attr:`write_concern` attribute is now read only. """ return self.__write_concern @property def read_preference(self): """Read only access to the read preference of this instance. .. versionchanged:: 3.0 The :attr:`read_preference` attribute is now read only. """ return self.__read_preference @property def read_concern(self): """Read only access to the :class:`~pymongo.read_concern.ReadConcern` of this instance. .. versionadded:: 3.2 """ return self.__read_concern pymongo-3.6.1/pymongo/client_options.py0000644000076600000240000002003213245621354020530 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Tools to parse mongo client options.""" from bson.codec_options import _parse_codec_options from pymongo.auth import _build_credentials_tuple from pymongo.common import validate_boolean from pymongo import common from pymongo.errors import ConfigurationError from pymongo.monitoring import _EventListeners from pymongo.pool import PoolOptions from pymongo.read_concern import ReadConcern from pymongo.read_preferences import (make_read_preference, read_pref_mode_from_name) from pymongo.ssl_support import get_ssl_context from pymongo.write_concern import WriteConcern def _parse_credentials(username, password, database, options): """Parse authentication credentials.""" mechanism = options.get('authmechanism', 'DEFAULT') if username is None and mechanism != 'MONGODB-X509': return None source = options.get('authsource', database or 'admin') return _build_credentials_tuple( mechanism, source, username, password, options) def _parse_read_preference(options): """Parse read preference options.""" if 'read_preference' in options: return options['read_preference'] name = options.get('readpreference', 'primary') mode = read_pref_mode_from_name(name) tags = options.get('readpreferencetags') max_staleness = options.get('maxstalenessseconds', -1) return make_read_preference(mode, tags, max_staleness) def _parse_write_concern(options): """Parse write concern options.""" concern = options.get('w') wtimeout = options.get('wtimeout') j = options.get('j', options.get('journal')) fsync = options.get('fsync') return WriteConcern(concern, wtimeout, j, fsync) def _parse_read_concern(options): """Parse read concern options.""" concern = options.get('readconcernlevel') return ReadConcern(concern) def _parse_ssl_options(options): """Parse ssl options.""" use_ssl = options.get('ssl') if use_ssl is not None: validate_boolean('ssl', use_ssl) certfile = options.get('ssl_certfile') keyfile = options.get('ssl_keyfile') passphrase = options.get('ssl_pem_passphrase') ca_certs = options.get('ssl_ca_certs') cert_reqs = options.get('ssl_cert_reqs') match_hostname = options.get('ssl_match_hostname', True) crlfile = options.get('ssl_crlfile') ssl_kwarg_keys = [k for k in options if k.startswith('ssl_') and options[k]] if use_ssl == False and ssl_kwarg_keys: raise ConfigurationError("ssl has not been enabled but the " "following ssl parameters have been set: " "%s. Please set `ssl=True` or remove." % ', '.join(ssl_kwarg_keys)) if ssl_kwarg_keys and use_ssl is None: # ssl options imply ssl = True use_ssl = True if use_ssl is True: ctx = get_ssl_context( certfile, keyfile, passphrase, ca_certs, cert_reqs, crlfile) return ctx, match_hostname return None, match_hostname def _parse_pool_options(options): """Parse connection pool options.""" max_pool_size = options.get('maxpoolsize', common.MAX_POOL_SIZE) min_pool_size = options.get('minpoolsize', common.MIN_POOL_SIZE) default_idle_seconds = common.validate_timeout_or_none( 'maxidletimems', common.MAX_IDLE_TIME_MS) max_idle_time_seconds = options.get('maxidletimems', default_idle_seconds) if max_pool_size is not None and min_pool_size > max_pool_size: raise ValueError("minPoolSize must be smaller or equal to maxPoolSize") connect_timeout = options.get('connecttimeoutms', common.CONNECT_TIMEOUT) socket_keepalive = options.get('socketkeepalive', True) socket_timeout = options.get('sockettimeoutms') wait_queue_timeout = options.get('waitqueuetimeoutms') wait_queue_multiple = options.get('waitqueuemultiple') event_listeners = options.get('event_listeners') appname = options.get('appname') ssl_context, ssl_match_hostname = _parse_ssl_options(options) return PoolOptions(max_pool_size, min_pool_size, max_idle_time_seconds, connect_timeout, socket_timeout, wait_queue_timeout, wait_queue_multiple, ssl_context, ssl_match_hostname, socket_keepalive, _EventListeners(event_listeners), appname) class ClientOptions(object): """ClientOptions""" def __init__(self, username, password, database, options): self.__options = options self.__codec_options = _parse_codec_options(options) self.__credentials = _parse_credentials( username, password, database, options) self.__local_threshold_ms = options.get( 'localthresholdms', common.LOCAL_THRESHOLD_MS) # self.__server_selection_timeout is in seconds. Must use full name for # common.SERVER_SELECTION_TIMEOUT because it is set directly by tests. self.__server_selection_timeout = options.get( 'serverselectiontimeoutms', common.SERVER_SELECTION_TIMEOUT) self.__pool_options = _parse_pool_options(options) self.__read_preference = _parse_read_preference(options) self.__replica_set_name = options.get('replicaset') self.__write_concern = _parse_write_concern(options) self.__read_concern = _parse_read_concern(options) self.__connect = options.get('connect') self.__heartbeat_frequency = options.get( 'heartbeatfrequencyms', common.HEARTBEAT_FREQUENCY) self.__retry_writes = options.get('retrywrites', common.RETRY_WRITES) @property def _options(self): """The original options used to create this ClientOptions.""" return self.__options @property def connect(self): """Whether to begin discovering a MongoDB topology automatically.""" return self.__connect @property def codec_options(self): """A :class:`~bson.codec_options.CodecOptions` instance.""" return self.__codec_options @property def credentials(self): """A :class:`~pymongo.auth.MongoCredentials` instance or None.""" return self.__credentials @property def local_threshold_ms(self): """The local threshold for this instance.""" return self.__local_threshold_ms @property def server_selection_timeout(self): """The server selection timeout for this instance in seconds.""" return self.__server_selection_timeout @property def heartbeat_frequency(self): """The monitoring frequency in seconds.""" return self.__heartbeat_frequency @property def pool_options(self): """A :class:`~pymongo.pool.PoolOptions` instance.""" return self.__pool_options @property def read_preference(self): """A read preference instance.""" return self.__read_preference @property def replica_set_name(self): """Replica set name or None.""" return self.__replica_set_name @property def write_concern(self): """A :class:`~pymongo.write_concern.WriteConcern` instance.""" return self.__write_concern @property def read_concern(self): """A :class:`~pymongo.read_concern.ReadConcern` instance.""" return self.__read_concern @property def retry_writes(self): """If this instance should retry supported write operations.""" return self.__retry_writes pymongo-3.6.1/pymongo/mongo_client.py0000644000076600000240000021741313245621354020167 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Tools for connecting to MongoDB. .. seealso:: :doc:`/examples/high_availability` for examples of connecting to replica sets or sets of mongos servers. To get a :class:`~pymongo.database.Database` instance from a :class:`MongoClient` use either dictionary-style or attribute-style access: .. doctest:: >>> from pymongo import MongoClient >>> c = MongoClient() >>> c.test_database Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), u'test_database') >>> c['test-database'] Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), u'test-database') """ import contextlib import datetime import threading import warnings import weakref from collections import defaultdict from bson.codec_options import DEFAULT_CODEC_OPTIONS from bson.py3compat import (integer_types, string_type) from bson.son import SON from pymongo import (common, database, helpers, message, periodic_executor, uri_parser, client_session) from pymongo.client_options import ClientOptions from pymongo.command_cursor import CommandCursor from pymongo.cursor_manager import CursorManager from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure, InvalidOperation, NetworkTimeout, NotMasterError, OperationFailure, PyMongoError, ServerSelectionTimeoutError) from pymongo.read_preferences import ReadPreference from pymongo.server_selectors import (writable_preferred_server_selector, writable_server_selector) from pymongo.server_type import SERVER_TYPE from pymongo.topology import Topology from pymongo.topology_description import TOPOLOGY_TYPE from pymongo.settings import TopologySettings from pymongo.write_concern import WriteConcern class MongoClient(common.BaseObject): HOST = "localhost" PORT = 27017 # Define order to retrieve options from ClientOptions for __repr__. # No host/port; these are retrieved from TopologySettings. _constructor_args = ('document_class', 'tz_aware', 'connect') def __init__( self, host=None, port=None, document_class=dict, tz_aware=None, connect=None, **kwargs): """Client for a MongoDB instance, a replica set, or a set of mongoses. The client object is thread-safe and has connection-pooling built in. If an operation fails because of a network error, :class:`~pymongo.errors.ConnectionFailure` is raised and the client reconnects in the background. Application code should handle this exception (recognizing that the operation failed) and then continue to execute. The `host` parameter can be a full `mongodb URI `_, in addition to a simple hostname. It can also be a list of hostnames or URIs. Any port specified in the host string(s) will override the `port` parameter. If multiple mongodb URIs containing database or auth information are passed, the last database, username, and password present will be used. For username and passwords reserved characters like ':', '/', '+' and '@' must be percent encoded following RFC 2396:: try: # Python 3.x from urllib.parse import quote_plus except ImportError: # Python 2.x from urllib import quote_plus uri = "mongodb://%s:%s@%s" % ( quote_plus(user), quote_plus(password), host) client = MongoClient(uri) Unix domain sockets are also supported. The socket path must be percent encoded in the URI:: uri = "mongodb://%s:%s@%s" % ( quote_plus(user), quote_plus(password), quote_plus(socket_path)) client = MongoClient(uri) But not when passed as a simple hostname:: client = MongoClient('/tmp/mongodb-27017.sock') Starting with version 3.6, PyMongo supports mongodb+srv:// URIs. The URI must include one, and only one, hostname. The hostname will be resolved to one or more DNS `SRV records `_ which will be used as the seed list for connecting to the MongoDB deployment. When using SRV URIs, the `authSource` and `replicaSet` configuration options can be specified using `TXT records `_. See the `Initial DNS Seedlist Discovery spec `_ for more details. Note that the use of SRV URIs implicitly enables TLS support. Pass ssl=false in the URI to override. .. note:: MongoClient creation will block waiting for answers from DNS when mongodb+srv:// URIs are used. .. note:: Starting with version 3.0 the :class:`MongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :class:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor :class:`~pymongo.errors.ConfigurationError` if the user's credentials are wrong. Instead, the constructor returns immediately and launches the connection process on background threads. You can check if the server is available like this:: from pymongo.errors import ConnectionFailure client = MongoClient() try: # The ismaster command is cheap and does not require auth. client.admin.command('ismaster') except ConnectionFailure: print("Server not available") .. warning:: When using PyMongo in a multiprocessing context, please read :ref:`multiprocessing` first. :Parameters: - `host` (optional): hostname or IP address or Unix domain socket path of a single mongod or mongos instance to connect to, or a mongodb URI, or a list of hostnames / mongodb URIs. If `host` is an IPv6 literal it must be enclosed in '[' and ']' characters following the RFC2732 URL syntax (e.g. '[::1]' for localhost). Multihomed and round robin DNS addresses are **not** supported. - `port` (optional): port number on which to connect - `document_class` (optional): default class to use for documents returned from queries on this client - `tz_aware` (optional): if ``True``, :class:`~datetime.datetime` instances returned as values in a document by this :class:`MongoClient` will be timezone aware (otherwise they will be naive) - `connect` (optional): if ``True`` (the default), immediately begin connecting to MongoDB in the background. Otherwise connect on the first operation. | **Other optional parameters can be passed as keyword arguments:** - `maxPoolSize` (optional): The maximum allowable number of concurrent connections to each connected server. Requests to a server will block if there are `maxPoolSize` outstanding connections to the requested server. Defaults to 100. Cannot be 0. - `minPoolSize` (optional): The minimum required number of concurrent connections that the pool will maintain to each connected server. Default is 0. - `maxIdleTimeMS` (optional): The maximum number of milliseconds that a connection can remain idle in the pool before being removed and replaced. Defaults to `None` (no limit). - `socketTimeoutMS`: (integer or None) Controls how long (in milliseconds) the driver will wait for a response after sending an ordinary (non-monitoring) database operation before concluding that a network error has occurred. Defaults to ``None`` (no timeout). - `connectTimeoutMS`: (integer or None) Controls how long (in milliseconds) the driver will wait during server monitoring when connecting a new socket to a server before concluding the server is unavailable. Defaults to ``20000`` (20 seconds). - `serverSelectionTimeoutMS`: (integer) Controls how long (in milliseconds) the driver will wait to find an available, appropriate server to carry out a database operation; while it is waiting, multiple server monitoring operations may be carried out, each controlled by `connectTimeoutMS`. Defaults to ``30000`` (30 seconds). - `waitQueueTimeoutMS`: (integer or None) How long (in milliseconds) a thread will wait for a socket from the pool if the pool has no free sockets. Defaults to ``None`` (no timeout). - `waitQueueMultiple`: (integer or None) Multiplied by maxPoolSize to give the number of threads allowed to wait for a socket at one time. Defaults to ``None`` (no limit). - `heartbeatFrequencyMS`: (optional) The number of milliseconds between periodic server checks, or None to accept the default frequency of 10 seconds. - `appname`: (string or None) The name of the application that created this MongoClient instance. MongoDB 3.4 and newer will print this value in the server log upon establishing each connection. It is also recorded in the slow query log and profile collections. - `event_listeners`: a list or tuple of event listeners. See :mod:`~pymongo.monitoring` for details. - `retryWrites`: (boolean) Whether supported write operations executed within this MongoClient will be retried once after a network error on MongoDB 3.6+. Defaults to ``False``. The supported write operations are: - :meth:`~pymongo.collection.Collection.bulk_write`, as long as :class:`~pymongo.operations.UpdateMany` or :class:`~pymongo.operations.DeleteMany` are not included. - :meth:`~pymongo.collection.Collection.delete_one` - :meth:`~pymongo.collection.Collection.insert_one` - :meth:`~pymongo.collection.Collection.insert_many` - :meth:`~pymongo.collection.Collection.replace_one` - :meth:`~pymongo.collection.Collection.update_one` - :meth:`~pymongo.collection.Collection.find_one_and_delete` - :meth:`~pymongo.collection.Collection.find_one_and_replace` - :meth:`~pymongo.collection.Collection.find_one_and_update` Unsupported write operations include, but are not limited to, :meth:`~pymongo.collection.Collection.aggregate` using the ``$out`` pipeline operator and any operation with an unacknowledged write concern (e.g. {w: 0})). See https://github.com/mongodb/specifications/blob/master/source/retryable-writes/retryable-writes.rst - `socketKeepAlive`: (boolean) **DEPRECATED** Whether to send periodic keep-alive packets on connected sockets. Defaults to ``True``. Disabling it is not recommended, see https://docs.mongodb.com/manual/faq/diagnostics/#does-tcp-keepalive-time-affect-mongodb-deployments", | **Write Concern options:** | (Only set if passed. No default values.) - `w`: (integer or string) If this is a replica set, write operations will block until they have been replicated to the specified number or tagged set of servers. `w=` always includes the replica set primary (e.g. w=3 means write to the primary and wait until replicated to **two** secondaries). Passing w=0 **disables write acknowledgement** and all other write concern options. - `wtimeout`: (integer) Used in conjunction with `w`. Specify a value in milliseconds to control how long to wait for write propagation to complete. If replication does not complete in the given timeframe, a timeout exception is raised. - `j`: If ``True`` block until write operations have been committed to the journal. Cannot be used in combination with `fsync`. Prior to MongoDB 2.6 this option was ignored if the server was running without journaling. Starting with MongoDB 2.6 write operations will fail with an exception if this option is used when the server is running without journaling. - `fsync`: If ``True`` and the server is running without journaling, blocks until the server has synced all data files to disk. If the server is running with journaling, this acts the same as the `j` option, blocking until write operations have been committed to the journal. Cannot be used in combination with `j`. | **Replica set keyword arguments for connecting with a replica set - either directly or via a mongos:** - `replicaSet`: (string or None) The name of the replica set to connect to. The driver will verify that all servers it connects to match this name. Implies that the hosts specified are a seed list and the driver should attempt to find all members of the set. Defaults to ``None``. | **Read Preference:** - `readPreference`: The replica set read preference for this client. One of ``primary``, ``primaryPreferred``, ``secondary``, ``secondaryPreferred``, or ``nearest``. Defaults to ``primary``. - `readPreferenceTags`: Specifies a tag set as a comma-separated list of colon-separated key-value pairs. For example ``dc:ny,rack:1``. Defaults to ``None``. - `maxStalenessSeconds`: (integer) The maximum estimated length of time a replica set secondary can fall behind the primary in replication before it will no longer be selected for operations. Defaults to ``-1``, meaning no maximum. If maxStalenessSeconds is set, it must be a positive integer greater than or equal to 90 seconds. | **Authentication:** - `username`: A string. - `password`: A string. Although username and password must be percent-escaped in a MongoDB URI, they must not be percent-escaped when passed as parameters. In this example, both the space and slash special characters are passed as-is:: MongoClient(username="user name", password="pass/word") - `authSource`: The database to authenticate on. Defaults to the database specified in the URI, if provided, or to "admin". - `authMechanism`: See :data:`~pymongo.auth.MECHANISMS` for options. By default, use SCRAM-SHA-1 with MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response protocol) for older servers. - `authMechanismProperties`: Used to specify authentication mechanism specific options. To specify the service name for GSSAPI authentication pass authMechanismProperties='SERVICE_NAME:' .. seealso:: :doc:`/examples/authentication` | **SSL configuration:** - `ssl`: If ``True``, create the connection to the server using SSL. Defaults to ``False``. - `ssl_certfile`: The certificate file used to identify the local connection against mongod. Implies ``ssl=True``. Defaults to ``None``. - `ssl_keyfile`: The private keyfile used to identify the local connection against mongod. If included with the ``certfile`` then only the ``ssl_certfile`` is needed. Implies ``ssl=True``. Defaults to ``None``. - `ssl_pem_passphrase`: The password or passphrase for decrypting the private key in ``ssl_certfile`` or ``ssl_keyfile``. Only necessary if the private key is encrypted. Only supported by python 2.7.9+ (pypy 2.5.1+) and 3.3+. Defaults to ``None``. - `ssl_cert_reqs`: Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided. It must be one of the three values ``ssl.CERT_NONE`` (certificates ignored), ``ssl.CERT_REQUIRED`` (certificates required and validated), or ``ssl.CERT_OPTIONAL`` (the same as CERT_REQUIRED, unless the server was configured to use anonymous ciphers). If the value of this parameter is not ``ssl.CERT_NONE`` and a value is not provided for ``ssl_ca_certs`` PyMongo will attempt to load system provided CA certificates. If the python version in use does not support loading system CA certificates then the ``ssl_ca_certs`` parameter must point to a file of CA certificates. Implies ``ssl=True``. Defaults to ``ssl.CERT_REQUIRED`` if not provided and ``ssl=True``. - `ssl_ca_certs`: The ca_certs file contains a set of concatenated "certification authority" certificates, which are used to validate certificates passed from the other end of the connection. Implies ``ssl=True``. Defaults to ``None``. - `ssl_crlfile`: The path to a PEM or DER formatted certificate revocation list. Only supported by python 2.7.9+ (pypy 2.5.1+) and 3.4+. Defaults to ``None``. - `ssl_match_hostname`: If ``True`` (the default), and `ssl_cert_reqs` is not ``ssl.CERT_NONE``, enables hostname verification using the :func:`~ssl.match_hostname` function from python's :mod:`~ssl` module. Think very carefully before setting this to ``False`` as that could make your application vulnerable to man-in-the-middle attacks. | **Read Concern options:** | (If not set explicitly, this will use the server default) - `readConcernLevel`: (string) The read concern level specifies the level of isolation for read operations. For example, a read operation using a read concern level of ``majority`` will only return data that has been written to a majority of nodes. If the level is left unspecified, the server default will be used. .. mongodoc:: connections .. versionchanged:: 3.6 Added support for mongodb+srv:// URIs. Added the ``retryWrites`` keyword argument and URI option. .. versionchanged:: 3.5 Add ``username`` and ``password`` options. Document the ``authSource``, ``authMechanism``, and ``authMechanismProperties `` options. Deprecated the `socketKeepAlive` keyword argument and URI option. `socketKeepAlive` now defaults to ``True``. .. versionchanged:: 3.0 :class:`~pymongo.mongo_client.MongoClient` is now the one and only client class for a standalone server, mongos, or replica set. It includes the functionality that had been split into :class:`~pymongo.mongo_client.MongoReplicaSetClient`: it can connect to a replica set, discover all its members, and monitor the set for stepdowns, elections, and reconfigs. The :class:`~pymongo.mongo_client.MongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :class:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor :class:`~pymongo.errors.ConfigurationError` if the user's credentials are wrong. Instead, the constructor returns immediately and launches the connection process on background threads. Therefore the ``alive`` method is removed since it no longer provides meaningful information; even if the client is disconnected, it may discover a server in time to fulfill the next operation. In PyMongo 2.x, :class:`~pymongo.MongoClient` accepted a list of standalone MongoDB servers and used the first it could connect to:: MongoClient(['host1.com:27017', 'host2.com:27017']) A list of multiple standalones is no longer supported; if multiple servers are listed they must be members of the same replica set, or mongoses in the same sharded cluster. The behavior for a list of mongoses is changed from "high availability" to "load balancing". Before, the client connected to the lowest-latency mongos in the list, and used it until a network error prompted it to re-evaluate all mongoses' latencies and reconnect to one of them. In PyMongo 3, the client monitors its network latency to all the mongoses continuously, and distributes operations evenly among those with the lowest latency. See :ref:`mongos-load-balancing` for more information. The ``connect`` option is added. The ``start_request``, ``in_request``, and ``end_request`` methods are removed, as well as the ``auto_start_request`` option. The ``copy_database`` method is removed, see the :doc:`copy_database examples ` for alternatives. The :meth:`MongoClient.disconnect` method is removed; it was a synonym for :meth:`~pymongo.MongoClient.close`. :class:`~pymongo.mongo_client.MongoClient` no longer returns an instance of :class:`~pymongo.database.Database` for attribute names with leading underscores. You must use dict-style lookups instead:: client['__my_database__'] Not:: client.__my_database__ """ if host is None: host = self.HOST if isinstance(host, string_type): host = [host] if port is None: port = self.PORT if not isinstance(port, int): raise TypeError("port must be an instance of int") seeds = set() username = None password = None dbase = None opts = {} for entity in host: if "://" in entity: res = uri_parser.parse_uri(entity, port, warn=True) seeds.update(res["nodelist"]) username = res["username"] or username password = res["password"] or password dbase = res["database"] or dbase opts = res["options"] else: seeds.update(uri_parser.split_hosts(entity, port)) if not seeds: raise ConfigurationError("need to specify at least one host") # _pool_class, _monitor_class, and _condition_class are for deep # customization of PyMongo, e.g. Motor. pool_class = kwargs.pop('_pool_class', None) monitor_class = kwargs.pop('_monitor_class', None) condition_class = kwargs.pop('_condition_class', None) keyword_opts = kwargs keyword_opts['document_class'] = document_class if tz_aware is None: tz_aware = opts.get('tz_aware', False) if connect is None: connect = opts.get('connect', True) keyword_opts['tz_aware'] = tz_aware keyword_opts['connect'] = connect # Validate all keyword options. keyword_opts = dict(common.validate(k, v) for k, v in keyword_opts.items()) opts.update(keyword_opts) # Username and password passed as kwargs override user info in URI. username = opts.get("username", username) password = opts.get("password", password) if 'socketkeepalive' in opts: warnings.warn( "The socketKeepAlive option is deprecated. It now" "defaults to true and disabling it is not recommended, see " "https://docs.mongodb.com/manual/faq/diagnostics/" "#does-tcp-keepalive-time-affect-mongodb-deployments", DeprecationWarning, stacklevel=2) self.__options = options = ClientOptions( username, password, dbase, opts) self.__default_database_name = dbase self.__lock = threading.Lock() self.__cursor_manager = None self.__kill_cursors_queue = [] self._event_listeners = options.pool_options.event_listeners # Cache of existing indexes used by ensure_index ops. self.__index_cache = {} self.__index_cache_lock = threading.Lock() super(MongoClient, self).__init__(options.codec_options, options.read_preference, options.write_concern, options.read_concern) self.__all_credentials = {} creds = options.credentials if creds: self._cache_credentials(creds.source, creds) self._topology_settings = TopologySettings( seeds=seeds, replica_set_name=options.replica_set_name, pool_class=pool_class, pool_options=options.pool_options, monitor_class=monitor_class, condition_class=condition_class, local_threshold_ms=options.local_threshold_ms, server_selection_timeout=options.server_selection_timeout, heartbeat_frequency=options.heartbeat_frequency) self._topology = Topology(self._topology_settings) if connect: self._topology.open() def target(): client = self_ref() if client is None: return False # Stop the executor. MongoClient._process_periodic_tasks(client) return True executor = periodic_executor.PeriodicExecutor( interval=common.KILL_CURSOR_FREQUENCY, min_interval=0.5, target=target, name="pymongo_kill_cursors_thread") # We strongly reference the executor and it weakly references us via # this closure. When the client is freed, stop the executor soon. self_ref = weakref.ref(self, executor.close) self._kill_cursors_executor = executor executor.open() def _cache_credentials(self, source, credentials, connect=False): """Save a set of authentication credentials. The credentials are used to login a socket whenever one is created. If `connect` is True, verify the credentials on the server first. """ # Don't let other threads affect this call's data. all_credentials = self.__all_credentials.copy() if source in all_credentials: # Nothing to do if we already have these credentials. if credentials == all_credentials[source]: return raise OperationFailure('Another user is already authenticated ' 'to this database. You must logout first.') if connect: server = self._get_topology().select_server( writable_preferred_server_selector) # get_socket() logs out of the database if logged in with old # credentials, and logs in with new ones. with server.get_socket(all_credentials) as sock_info: sock_info.authenticate(credentials) # If several threads run _cache_credentials at once, last one wins. self.__all_credentials[source] = credentials def _purge_credentials(self, source): """Purge credentials from the authentication cache.""" self.__all_credentials.pop(source, None) def _cached(self, dbname, coll, index): """Test if `index` is cached.""" cache = self.__index_cache now = datetime.datetime.utcnow() with self.__index_cache_lock: return (dbname in cache and coll in cache[dbname] and index in cache[dbname][coll] and now < cache[dbname][coll][index]) def _cache_index(self, dbname, collection, index, cache_for): """Add an index to the index cache for ensure_index operations.""" now = datetime.datetime.utcnow() expire = datetime.timedelta(seconds=cache_for) + now with self.__index_cache_lock: if database not in self.__index_cache: self.__index_cache[dbname] = {} self.__index_cache[dbname][collection] = {} self.__index_cache[dbname][collection][index] = expire elif collection not in self.__index_cache[dbname]: self.__index_cache[dbname][collection] = {} self.__index_cache[dbname][collection][index] = expire else: self.__index_cache[dbname][collection][index] = expire def _purge_index(self, database_name, collection_name=None, index_name=None): """Purge an index from the index cache. If `index_name` is None purge an entire collection. If `collection_name` is None purge an entire database. """ with self.__index_cache_lock: if not database_name in self.__index_cache: return if collection_name is None: del self.__index_cache[database_name] return if not collection_name in self.__index_cache[database_name]: return if index_name is None: del self.__index_cache[database_name][collection_name] return if index_name in self.__index_cache[database_name][collection_name]: del self.__index_cache[database_name][collection_name][index_name] def _server_property(self, attr_name): """An attribute of the current server's description. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. Not threadsafe if used multiple times in a single method, since the server may change. In such cases, store a local reference to a ServerDescription first, then use its properties. """ server = self._topology.select_server( writable_server_selector) return getattr(server.description, attr_name) @property def event_listeners(self): """The event listeners registered for this client. See :mod:`~pymongo.monitoring` for details. """ return self._event_listeners.event_listeners @property def address(self): """(host, port) of the current standalone, primary, or mongos, or None. Accessing :attr:`address` raises :exc:`~.errors.InvalidOperation` if the client is load-balancing among mongoses, since there is no single address. Use :attr:`nodes` instead. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. .. versionadded:: 3.0 """ topology_type = self._topology._description.topology_type if topology_type == TOPOLOGY_TYPE.Sharded: raise InvalidOperation( 'Cannot use "address" property when load balancing among' ' mongoses, use "nodes" instead.') if topology_type not in (TOPOLOGY_TYPE.ReplicaSetWithPrimary, TOPOLOGY_TYPE.Single): return None return self._server_property('address') @property def primary(self): """The (host, port) of the current primary of the replica set. Returns ``None`` if this client is not connected to a replica set, there is no primary, or this client was created without the `replicaSet` option. .. versionadded:: 3.0 MongoClient gained this property in version 3.0 when MongoReplicaSetClient's functionality was merged in. """ return self._topology.get_primary() @property def secondaries(self): """The secondary members known to this client. A sequence of (host, port) pairs. Empty if this client is not connected to a replica set, there are no visible secondaries, or this client was created without the `replicaSet` option. .. versionadded:: 3.0 MongoClient gained this property in version 3.0 when MongoReplicaSetClient's functionality was merged in. """ return self._topology.get_secondaries() @property def arbiters(self): """Arbiters in the replica set. A sequence of (host, port) pairs. Empty if this client is not connected to a replica set, there are no arbiters, or this client was created without the `replicaSet` option. """ return self._topology.get_arbiters() @property def is_primary(self): """If this client is connected to a server that can accept writes. True if the current server is a standalone, mongos, or the primary of a replica set. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. """ return self._server_property('is_writable') @property def is_mongos(self): """If this client is connected to mongos. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available.. """ return self._server_property('server_type') == SERVER_TYPE.Mongos @property def max_pool_size(self): """The maximum allowable number of concurrent connections to each connected server. Requests to a server will block if there are `maxPoolSize` outstanding connections to the requested server. Defaults to 100. Cannot be 0. When a server's pool has reached `max_pool_size`, operations for that server block waiting for a socket to be returned to the pool. If ``waitQueueTimeoutMS`` is set, a blocked operation will raise :exc:`~pymongo.errors.ConnectionFailure` after a timeout. By default ``waitQueueTimeoutMS`` is not set. """ return self.__options.pool_options.max_pool_size @property def min_pool_size(self): """The minimum required number of concurrent connections that the pool will maintain to each connected server. Default is 0. """ return self.__options.pool_options.min_pool_size @property def max_idle_time_ms(self): """The maximum number of milliseconds that a connection can remain idle in the pool before being removed and replaced. Defaults to `None` (no limit). """ seconds = self.__options.pool_options.max_idle_time_seconds if seconds is None: return None return 1000 * seconds @property def nodes(self): """Set of all currently connected servers. .. warning:: When connected to a replica set the value of :attr:`nodes` can change over time as :class:`MongoClient`'s view of the replica set changes. :attr:`nodes` can also be an empty set when :class:`MongoClient` is first instantiated and hasn't yet connected to any servers, or a network partition causes it to lose connection to all servers. """ description = self._topology.description return frozenset(s.address for s in description.known_servers) @property def max_bson_size(self): """The largest BSON object the connected server accepts in bytes. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. """ return self._server_property('max_bson_size') @property def max_message_size(self): """The largest message the connected server accepts in bytes. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. """ return self._server_property('max_message_size') @property def max_write_batch_size(self): """The maxWriteBatchSize reported by the server. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. Returns a default value when connected to server versions prior to MongoDB 2.6. """ return self._server_property('max_write_batch_size') @property def local_threshold_ms(self): """The local threshold for this instance.""" return self.__options.local_threshold_ms @property def server_selection_timeout(self): """The server selection timeout for this instance in seconds.""" return self.__options.server_selection_timeout @property def retry_writes(self): """If this instance should retry supported write operations.""" return self.__options.retry_writes def _is_writable(self): """Attempt to connect to a writable server, or return False. """ topology = self._get_topology() # Starts monitors if necessary. try: svr = topology.select_server(writable_server_selector) # When directly connected to a secondary, arbiter, etc., # select_server returns it, whatever the selector. Check # again if the server is writable. return svr.description.is_writable except ConnectionFailure: return False def _end_sessions(self, session_ids): """Send endSessions command(s) with the given session ids.""" try: # Use SocketInfo.command directly to avoid implicitly creating # another session. with self._socket_for_reads( ReadPreference.PRIMARY_PREFERRED) as (sock_info, slave_ok): if not sock_info.supports_sessions: return for i in range(0, len(session_ids), common._MAX_END_SESSIONS): spec = SON([('endSessions', session_ids[i:i + common._MAX_END_SESSIONS])]) sock_info.command( 'admin', spec, slave_ok=slave_ok, client=self) except PyMongoError: # Drivers MUST ignore any errors returned by the endSessions # command. pass def close(self): """Cleanup client resources and disconnect from MongoDB. On MongoDB >= 3.6, end all server sessions created by this client by sending one or more endSessions commands. Close all sockets in the connection pools and stop the monitor threads. If this instance is used again it will be automatically re-opened and the threads restarted. .. versionchanged:: 3.6 End all server sessions created by this client. """ session_ids = self._topology.pop_all_sessions() if session_ids: self._end_sessions(session_ids) # Run _process_periodic_tasks to send pending killCursor requests # before closing the topology. self._process_periodic_tasks() self._topology.close() def set_cursor_manager(self, manager_class): """DEPRECATED - Set this client's cursor manager. Raises :class:`TypeError` if `manager_class` is not a subclass of :class:`~pymongo.cursor_manager.CursorManager`. A cursor manager handles closing cursors. Different managers can implement different policies in terms of when to actually kill a cursor that has been closed. :Parameters: - `manager_class`: cursor manager to use .. versionchanged:: 3.3 Deprecated, for real this time. .. versionchanged:: 3.0 Undeprecated. """ warnings.warn( "set_cursor_manager is Deprecated", DeprecationWarning, stacklevel=2) manager = manager_class(self) if not isinstance(manager, CursorManager): raise TypeError("manager_class must be a subclass of " "CursorManager") self.__cursor_manager = manager def _get_topology(self): """Get the internal :class:`~pymongo.topology.Topology` object. If this client was created with "connect=False", calling _get_topology launches the connection process in the background. """ self._topology.open() return self._topology @contextlib.contextmanager def _get_socket(self, server): try: with server.get_socket(self.__all_credentials) as sock_info: yield sock_info except NetworkTimeout: # The socket has been closed. Don't reset the server. # Server Discovery And Monitoring Spec: "When an application # operation fails because of any network error besides a socket # timeout...." raise except NotMasterError: # "When the client sees a "not master" error it MUST replace the # server's description with type Unknown. It MUST request an # immediate check of the server." self._reset_server_and_request_check(server.description.address) raise except ConnectionFailure: # "Client MUST replace the server's description with type Unknown # ... MUST NOT request an immediate check of the server." self.__reset_server(server.description.address) raise def _socket_for_writes(self): server = self._get_topology().select_server(writable_server_selector) return self._get_socket(server) @contextlib.contextmanager def _socket_for_reads(self, read_preference): preference = read_preference or ReadPreference.PRIMARY # Get a socket for a server matching the read preference, and yield # sock_info, slave_ok. Server Selection Spec: "slaveOK must be sent to # mongods with topology type Single. If the server type is Mongos, # follow the rules for passing read preference to mongos, even for # topology type Single." # Thread safe: if the type is single it cannot change. topology = self._get_topology() single = topology.description.topology_type == TOPOLOGY_TYPE.Single server = topology.select_server(read_preference) with self._get_socket(server) as sock_info: slave_ok = (single and not sock_info.is_mongos) or ( preference != ReadPreference.PRIMARY) yield sock_info, slave_ok def _send_message_with_response(self, operation, read_preference=None, exhaust=False, address=None): """Send a message to MongoDB and return a Response. :Parameters: - `operation`: a _Query or _GetMore object. - `read_preference` (optional): A ReadPreference. - `exhaust` (optional): If True, the socket used stays checked out. It is returned along with its Pool in the Response. - `address` (optional): Optional address when sending a message to a specific server, used for getMore. """ with self.__lock: # If needed, restart kill-cursors thread after a fork. self._kill_cursors_executor.open() topology = self._get_topology() if address: server = topology.select_server_by_address(address) if not server: raise AutoReconnect('server %s:%d no longer available' % address) else: selector = read_preference or writable_server_selector server = topology.select_server(selector) # A _Query's slaveOk bit is already set for queries with non-primary # read preference. If this is a direct connection to a mongod, override # and *always* set the slaveOk bit. See bullet point 2 in # server-selection.rst#topology-type-single. set_slave_ok = ( topology.description.topology_type == TOPOLOGY_TYPE.Single and server.description.server_type != SERVER_TYPE.Mongos) return self._reset_on_error( server, server.send_message_with_response, operation, set_slave_ok, self.__all_credentials, self._event_listeners, exhaust) def _reset_on_error(self, server, func, *args, **kwargs): """Execute an operation. Reset the server on network error. Returns fn()'s return value on success. On error, clears the server's pool and marks the server Unknown. Re-raises any exception thrown by fn(). """ try: return func(*args, **kwargs) except NetworkTimeout: # The socket has been closed. Don't reset the server. raise except ConnectionFailure: self.__reset_server(server.description.address) raise def _retry_with_session(self, retryable, func, session, bulk): """Execute an operation with at most one consecutive retries Returns func()'s return value on success. On error retries the same command once. Re-raises any exception thrown by func(). """ retryable = retryable and self.retry_writes last_error = None retrying = False def is_retrying(): return bulk.retrying if bulk else retrying while True: try: server = self._get_topology().select_server( writable_server_selector) supports_session = ( session is not None and server.description.retryable_writes_supported) with self._get_socket(server) as sock_info: if retryable and not supports_session: if is_retrying(): # A retry is not possible because this server does # not support sessions raise the last error. raise last_error retryable = False if is_retrying(): # Reset the transaction id and retry the operation. session._retry_transaction_id() return func(session, sock_info, retryable) except ServerSelectionTimeoutError: if is_retrying(): # The application may think the write was never attempted # if we raise ServerSelectionTimeoutError on the retry # attempt. Raise the original exception instead. raise last_error # A ServerSelectionTimeoutError error indicates that there may # be a persistent outage. Attempting to retry in this case will # most likely be a waste of time. raise except ConnectionFailure as exc: if not retryable or is_retrying(): raise if bulk: bulk.retrying = True else: retrying = True last_error = exc def _retryable_write(self, retryable, func, session): """Internal retryable write helper.""" with self._tmp_session(session) as s: return self._retry_with_session(retryable, func, s, None) def __reset_server(self, address): """Clear our connection pool for a server and mark it Unknown.""" self._topology.reset_server(address) def _reset_server_and_request_check(self, address): """Clear our pool for a server, mark it Unknown, and check it soon.""" self._topology.reset_server_and_request_check(address) def __eq__(self, other): if isinstance(other, self.__class__): return self.address == other.address return NotImplemented def __ne__(self, other): return not self == other def _repr_helper(self): def option_repr(option, value): """Fix options whose __repr__ isn't usable in a constructor.""" if option == 'document_class': if value is dict: return 'document_class=dict' else: return 'document_class=%s.%s' % (value.__module__, value.__name__) if option in common.TIMEOUT_VALIDATORS and value is not None: return "%s=%s" % (option, int(value * 1000)) return '%s=%r' % (option, value) # Host first... options = ['host=%r' % [ '%s:%d' % (host, port) if port is not None else host for host, port in self._topology_settings.seeds]] # ... then everything in self._constructor_args... options.extend( option_repr(key, self.__options._options[key]) for key in self._constructor_args) # ... then everything else. options.extend( option_repr(key, self.__options._options[key]) for key in self.__options._options if key not in set(self._constructor_args) and key != 'username' and key != 'password') return ', '.join(options) def __repr__(self): return ("MongoClient(%s)" % (self._repr_helper(),)) def __getattr__(self, name): """Get a database by name. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. :Parameters: - `name`: the name of the database to get """ if name.startswith('_'): raise AttributeError( "MongoClient has no attribute %r. To access the %s" " database, use client[%r]." % (name, name, name)) return self.__getitem__(name) def __getitem__(self, name): """Get a database by name. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. :Parameters: - `name`: the name of the database to get """ return database.Database(self, name) def close_cursor(self, cursor_id, address=None): """Send a kill cursors message soon with the given id. Raises :class:`TypeError` if `cursor_id` is not an instance of ``(int, long)``. What closing the cursor actually means depends on this client's cursor manager. This method may be called from a :class:`~pymongo.cursor.Cursor` destructor during garbage collection, so it isn't safe to take a lock or do network I/O. Instead, we schedule the cursor to be closed soon on a background thread. :Parameters: - `cursor_id`: id of cursor to close - `address` (optional): (host, port) pair of the cursor's server. If it is not provided, the client attempts to close the cursor on the primary or standalone, or a mongos server. .. versionchanged:: 3.0 Added ``address`` parameter. """ if not isinstance(cursor_id, integer_types): raise TypeError("cursor_id must be an instance of (int, long)") if self.__cursor_manager is not None: self.__cursor_manager.close(cursor_id, address) else: self.__kill_cursors_queue.append((address, [cursor_id])) def _close_cursor_now(self, cursor_id, address=None, session=None): """Send a kill cursors message with the given id. What closing the cursor actually means depends on this client's cursor manager. If there is none, the cursor is closed synchronously on the current thread. """ if not isinstance(cursor_id, integer_types): raise TypeError("cursor_id must be an instance of (int, long)") if self.__cursor_manager is not None: self.__cursor_manager.close(cursor_id, address) else: try: self._kill_cursors( [cursor_id], address, self._get_topology(), session) except PyMongoError: # Make another attempt to kill the cursor later. self.__kill_cursors_queue.append((address, [cursor_id])) def kill_cursors(self, cursor_ids, address=None): """DEPRECATED - Send a kill cursors message soon with the given ids. Raises :class:`TypeError` if `cursor_ids` is not an instance of ``list``. :Parameters: - `cursor_ids`: list of cursor ids to kill - `address` (optional): (host, port) pair of the cursor's server. If it is not provided, the client attempts to close the cursor on the primary or standalone, or a mongos server. .. versionchanged:: 3.3 Deprecated. .. versionchanged:: 3.0 Now accepts an `address` argument. Schedules the cursors to be closed on a background thread instead of sending the message immediately. """ warnings.warn( "kill_cursors is deprecated.", DeprecationWarning, stacklevel=2) if not isinstance(cursor_ids, list): raise TypeError("cursor_ids must be a list") # "Atomic", needs no lock. self.__kill_cursors_queue.append((address, cursor_ids)) def _kill_cursors(self, cursor_ids, address, topology, session): """Send a kill cursors message with the given ids.""" listeners = self._event_listeners publish = listeners.enabled_for_commands if address: # address could be a tuple or _CursorAddress, but # select_server_by_address needs (host, port). server = topology.select_server_by_address(tuple(address)) else: # Application called close_cursor() with no address. server = topology.select_server(writable_server_selector) try: namespace = address.namespace db, coll = namespace.split('.', 1) except AttributeError: namespace = None db = coll = "OP_KILL_CURSORS" spec = SON([('killCursors', coll), ('cursors', cursor_ids)]) with server.get_socket(self.__all_credentials) as sock_info: if sock_info.max_wire_version >= 4 and namespace is not None: sock_info.command(db, spec, session=session, client=self) else: if publish: start = datetime.datetime.now() request_id, msg = message.kill_cursors(cursor_ids) if publish: duration = datetime.datetime.now() - start # Here and below, address could be a tuple or # _CursorAddress. We always want to publish a # tuple to match the rest of the monitoring # API. listeners.publish_command_start( spec, db, request_id, tuple(address)) start = datetime.datetime.now() try: sock_info.send_message(msg, 0) except Exception as exc: if publish: dur = ((datetime.datetime.now() - start) + duration) listeners.publish_command_failure( dur, message._convert_exception(exc), 'killCursors', request_id, tuple(address)) raise if publish: duration = ((datetime.datetime.now() - start) + duration) # OP_KILL_CURSORS returns no reply, fake one. reply = {'cursorsUnknown': cursor_ids, 'ok': 1} listeners.publish_command_success( duration, reply, 'killCursors', request_id, tuple(address)) # This method is run periodically by a background thread. def _process_periodic_tasks(self): """Process any pending kill cursors requests and maintain connection pool parameters.""" address_to_cursor_ids = defaultdict(list) # Other threads or the GC may append to the queue concurrently. while True: try: address, cursor_ids = self.__kill_cursors_queue.pop() except IndexError: break address_to_cursor_ids[address].extend(cursor_ids) # Don't re-open topology if it's closed and there's no pending cursors. if address_to_cursor_ids: topology = self._get_topology() for address, cursor_ids in address_to_cursor_ids.items(): try: self._kill_cursors( cursor_ids, address, topology, session=None) except Exception: helpers._handle_exception() try: self._topology.update_pool() except Exception: helpers._handle_exception() def start_session(self, causal_consistency=True): """Start a logical session. This method takes the same parameters as :class:`~pymongo.client_session.SessionOptions`. See the :mod:`~pymongo.client_session` module for details and examples. Requires MongoDB 3.6. It is an error to call :meth:`start_session` if this client has been authenticated to multiple databases using the deprecated method :meth:`~pymongo.database.Database.authenticate`. A :class:`~pymongo.client_session.ClientSession` may only be used with the MongoClient that started it. :Returns: An instance of :class:`~pymongo.client_session.ClientSession`. .. versionadded:: 3.6 """ # Driver Sessions Spec: "If startSession is called when multiple users # are authenticated drivers MUST raise an error with the error message # 'Cannot call startSession when multiple users are authenticated.'" authset = set(self.__all_credentials.values()) if len(authset) > 1: raise InvalidOperation("Cannot call start_session when" " multiple users are authenticated") # Raises ConfigurationError if sessions are not supported. server_session = self._get_server_session() opts = client_session.SessionOptions( causal_consistency=causal_consistency) return client_session.ClientSession( self, server_session, opts, authset) def _get_server_session(self): """Internal: start or resume a _ServerSession.""" return self._topology.get_server_session() def _return_server_session(self, server_session, lock): """Internal: return a _ServerSession to the pool.""" return self._topology.return_server_session(server_session, lock) def _ensure_session(self, session=None): """If provided session is None, lend a temporary session.""" if session: return session try: # Don't make implied sessions causally consistent. Applications # should always opt-in. return self.start_session(causal_consistency=False) except (ConfigurationError, InvalidOperation): # Sessions not supported, or multiple users authenticated. return None @contextlib.contextmanager def _tmp_session(self, session, close=True): """If provided session is None, lend a temporary session.""" if session: # Don't call end_session. yield session return s = self._ensure_session(session) if s and close: with s: # Call end_session when we exit this scope. yield s elif s: try: # Only call end_session on error. yield s except Exception: s.end_session() raise else: yield None def _send_cluster_time(self, command, session): topology_time = self._topology.max_cluster_time() session_time = session.cluster_time if session else None if topology_time and session_time: if topology_time['clusterTime'] > session_time['clusterTime']: cluster_time = topology_time else: cluster_time = session_time else: cluster_time = topology_time or session_time if cluster_time: command['$clusterTime'] = cluster_time def _receive_cluster_time(self, reply, session): cluster_time = reply.get('$clusterTime') self._topology.receive_cluster_time(cluster_time) if session is not None: session._advance_cluster_time(cluster_time) session._advance_operation_time(reply.get("operationTime")) def server_info(self, session=None): """Get information about the MongoDB server we're connected to. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. """ return self.admin.command("buildinfo", read_preference=ReadPreference.PRIMARY, session=session) def list_databases(self, session=None, **kwargs): """Get a cursor over the databases of the connected server. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): Optional parameters of the `listDatabases command `_ can be passed as keyword arguments to this method. The supported options differ by server version. :Returns: An instance of :class:`~pymongo.command_cursor.CommandCursor`. .. versionadded:: 3.6 """ cmd = SON([("listDatabases", 1)]) cmd.update(kwargs) res = self._database_default_options( "admin").command(cmd, session=session) # listDatabases doesn't return a cursor (yet). Fake one. cursor = { "id": 0, "firstBatch": res["databases"], "ns": "admin.$cmd", } return CommandCursor(self.admin["$cmd"], cursor, None) def list_database_names(self, session=None): """Get a list of the names of all databases on the connected server. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. """ return [doc["name"] for doc in self.list_databases(session, nameOnly=True)] database_names = list_database_names def drop_database(self, name_or_database, session=None): """Drop a database. Raises :class:`TypeError` if `name_or_database` is not an instance of :class:`basestring` (:class:`str` in python 3) or :class:`~pymongo.database.Database`. :Parameters: - `name_or_database`: the name of a database to drop, or a :class:`~pymongo.database.Database` instance representing the database to drop - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. .. note:: The :attr:`~pymongo.mongo_client.MongoClient.write_concern` of this client is automatically applied to this operation when using MongoDB >= 3.4. .. versionchanged:: 3.4 Apply this client's write concern automatically to this operation when connected to MongoDB >= 3.4. """ name = name_or_database if isinstance(name, database.Database): name = name.name if not isinstance(name, string_type): raise TypeError("name_or_database must be an instance " "of %s or a Database" % (string_type.__name__,)) self._purge_index(name) with self._socket_for_reads( ReadPreference.PRIMARY) as (sock_info, slave_ok): self[name]._command( sock_info, "dropDatabase", slave_ok=slave_ok, read_preference=ReadPreference.PRIMARY, write_concern=self.write_concern, parse_write_concern_error=True, session=session) def get_default_database(self): """DEPRECATED - Get the database named in the MongoDB connection URI. >>> uri = 'mongodb://host/my_database' >>> client = MongoClient(uri) >>> db = client.get_default_database() >>> assert db.name == 'my_database' >>> db = client.get_database() >>> assert db.name == 'my_database' Useful in scripts where you want to choose which database to use based only on the URI in a configuration file. .. versionchanged:: 3.5 Deprecated, use :meth:`get_database` instead. """ warnings.warn("get_default_database is deprecated. Use get_database " "instead.", DeprecationWarning, stacklevel=2) if self.__default_database_name is None: raise ConfigurationError('No default database defined') return self[self.__default_database_name] def get_database(self, name=None, codec_options=None, read_preference=None, write_concern=None, read_concern=None): """Get a :class:`~pymongo.database.Database` with the given name and options. Useful for creating a :class:`~pymongo.database.Database` with different codec options, read preference, and/or write concern from this :class:`MongoClient`. >>> client.read_preference Primary() >>> db1 = client.test >>> db1.read_preference Primary() >>> from pymongo import ReadPreference >>> db2 = client.get_database( ... 'test', read_preference=ReadPreference.SECONDARY) >>> db2.read_preference Secondary(tag_sets=None) :Parameters: - `name` (optional): The name of the database - a string. If ``None`` (the default) the database named in the MongoDB connection URI is returned. - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`MongoClient` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`MongoClient` is used. See :mod:`~pymongo.read_preferences` for options. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`MongoClient` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`MongoClient` is used. .. versionchanged:: 3.5 The `name` parameter is now optional, defaulting to the database named in the MongoDB connection URI. """ if name is None: if self.__default_database_name is None: raise ConfigurationError('No default database defined') name = self.__default_database_name return database.Database( self, name, codec_options, read_preference, write_concern, read_concern) def _database_default_options(self, name): """Get a Database instance with the default settings.""" return self.get_database( name, codec_options=DEFAULT_CODEC_OPTIONS, read_preference=ReadPreference.PRIMARY, write_concern=WriteConcern()) @property def is_locked(self): """Is this server locked? While locked, all write operations are blocked, although read operations may still be allowed. Use :meth:`unlock` to unlock. """ ops = self._database_default_options('admin').current_op() return bool(ops.get('fsyncLock', 0)) def fsync(self, **kwargs): """Flush all pending writes to datafiles. Optional parameters can be passed as keyword arguments: - `lock`: If True lock the server to disallow writes. - `async`: If True don't block while synchronizing. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. note:: Starting with Python 3.7 `async` is a reserved keyword. The async option to the fsync command can be passed using a dictionary instead:: options = {'async': True} client.fsync(**options) .. versionchanged:: 3.6 Added ``session`` parameter. .. warning:: `async` and `lock` can not be used together. .. warning:: MongoDB does not support the `async` option on Windows and will raise an exception on that platform. """ self.admin.command("fsync", read_preference=ReadPreference.PRIMARY, **kwargs) def unlock(self, session=None): """Unlock a previously locked server. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. """ cmd = SON([("fsyncUnlock", 1)]) with self._socket_for_writes() as sock_info: if sock_info.max_wire_version >= 4: try: with self._tmp_session(session) as s: sock_info.command( "admin", cmd, session=s, client=self) except OperationFailure as exc: # Ignore "DB not locked" to replicate old behavior if exc.code != 125: raise else: message._first_batch(sock_info, "admin", "$cmd.sys.unlock", {}, -1, True, self.codec_options, ReadPreference.PRIMARY, cmd, self._event_listeners) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def __iter__(self): return self def __next__(self): raise TypeError("'MongoClient' object is not iterable") next = __next__ pymongo-3.6.1/pymongo/network.py0000644000076600000240000002117513245621354017201 0ustar shanestaff00000000000000# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Internal network layer helper methods.""" import datetime import errno import select import struct import threading _HAS_POLL = True _EVENT_MASK = 0 try: from select import poll _EVENT_MASK = ( select.POLLIN | select.POLLPRI | select.POLLERR | select.POLLHUP) except ImportError: _HAS_POLL = False try: from select import error as _SELECT_ERROR except ImportError: _SELECT_ERROR = OSError from pymongo import helpers, message from pymongo.common import MAX_MESSAGE_SIZE from pymongo.errors import (AutoReconnect, NotMasterError, OperationFailure, ProtocolError) from pymongo.message import _OpReply _UNPACK_HEADER = struct.Struct(" max_bson_size + message._COMMAND_OVERHEAD): message._raise_document_too_large( name, size, max_bson_size + message._COMMAND_OVERHEAD) if publish: encoding_duration = datetime.datetime.now() - start listeners.publish_command_start(orig, dbname, request_id, address) start = datetime.datetime.now() try: sock.sendall(msg) reply = receive_message(sock, request_id) unpacked_docs = reply.unpack_response(codec_options=codec_options) response_doc = unpacked_docs[0] if client: client._receive_cluster_time(response_doc, session) if check: helpers._check_command_response( response_doc, None, allowable_errors, parse_write_concern_error=parse_write_concern_error) except Exception as exc: if publish: duration = (datetime.datetime.now() - start) + encoding_duration if isinstance(exc, (NotMasterError, OperationFailure)): failure = exc.details else: failure = message._convert_exception(exc) listeners.publish_command_failure( duration, failure, name, request_id, address) raise if publish: duration = (datetime.datetime.now() - start) + encoding_duration listeners.publish_command_success( duration, response_doc, name, request_id, address) return response_doc def receive_message(sock, request_id, max_message_size=MAX_MESSAGE_SIZE): """Receive a raw BSON message or raise socket.error.""" # Ignore the response's request id. length, _, response_to, op_code = _UNPACK_HEADER( _receive_data_on_socket(sock, 16)) if op_code != _OpReply.OP_CODE: raise ProtocolError("Got opcode %r but expected " "%r" % (op_code, _OpReply.OP_CODE)) # No request_id for exhaust cursor "getMore". if request_id is not None: if request_id != response_to: raise ProtocolError("Got response id %r but expected " "%r" % (response_to, request_id)) if length <= 16: raise ProtocolError("Message length (%r) not longer than standard " "message header size (16)" % (length,)) if length > max_message_size: raise ProtocolError("Message length (%r) is larger than server max " "message size (%r)" % (length, max_message_size)) return _OpReply.unpack(_receive_data_on_socket(sock, length - 16)) def _receive_data_on_socket(sock, length): msg = b"" while length: try: chunk = sock.recv(length) except (IOError, OSError) as exc: if _errno_from_exception(exc) == errno.EINTR: continue raise if chunk == b"": raise AutoReconnect("connection closed") length -= len(chunk) msg += chunk return msg def _errno_from_exception(exc): if hasattr(exc, 'errno'): return exc.errno elif exc.args: return exc.args[0] else: return None class SocketChecker(object): def __init__(self): if _HAS_POLL: self._lock = threading.Lock() self._poller = poll() else: self._lock = None self._poller = None def socket_closed(self, sock): """Return True if we know socket has been closed, False otherwise. """ while True: try: if self._poller: with self._lock: self._poller.register(sock, _EVENT_MASK) try: rd = self._poller.poll(0) finally: self._poller.unregister(sock) else: rd, _, _ = select.select([sock], [], [], 0) except (RuntimeError, KeyError): # RuntimeError is raised during a concurrent poll. KeyError # is raised by unregister if the socket is not in the poller. # These errors should not be possible since we protect the # poller with a mutex. raise except ValueError: # ValueError is raised by register/unregister/select if the # socket file descriptor is negative or outside the range for # select (> 1023). return True except (_SELECT_ERROR, IOError) as exc: if _errno_from_exception(exc) in (errno.EINTR, errno.EAGAIN): continue return True except Exception: # Any other exceptions should be attributed to a closed # or invalid socket. return True return len(rd) > 0 pymongo-3.6.1/pymongo/topology.py0000644000076600000240000005777313245621354017401 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Internal class to monitor a topology of one or more servers.""" import os import random import threading import warnings import weakref from bson.py3compat import itervalues, PY3 if PY3: import queue as Queue else: import Queue from pymongo import common from pymongo import periodic_executor from pymongo.pool import PoolOptions from pymongo.topology_description import (updated_topology_description, TOPOLOGY_TYPE, TopologyDescription) from pymongo.errors import ServerSelectionTimeoutError, ConfigurationError from pymongo.monotonic import time as _time from pymongo.server import Server from pymongo.server_selectors import (any_server_selector, arbiter_server_selector, secondary_server_selector, readable_server_selector, writable_server_selector, Selection) from pymongo.client_session import _ServerSessionPool def process_events_queue(queue_ref): q = queue_ref() if not q: return False # Cancel PeriodicExecutor. while True: try: event = q.get_nowait() except Queue.Empty: break else: fn, args = event fn(*args) return True # Continue PeriodicExecutor. class Topology(object): """Monitor a topology of one or more servers.""" def __init__(self, topology_settings): self._topology_id = topology_settings._topology_id self._listeners = topology_settings._pool_options.event_listeners pub = self._listeners is not None self._publish_server = pub and self._listeners.enabled_for_server self._publish_tp = pub and self._listeners.enabled_for_topology # Create events queue if there are publishers. self._events = None self._events_thread = None if self._publish_server or self._publish_tp: self._events = Queue.Queue(maxsize=100) if self._publish_tp: self._events.put((self._listeners.publish_topology_opened, (self._topology_id,))) self._settings = topology_settings topology_description = TopologyDescription( topology_settings.get_topology_type(), topology_settings.get_server_descriptions(), topology_settings.replica_set_name, None, None, topology_settings) self._description = topology_description if self._publish_tp: initial_td = TopologyDescription(TOPOLOGY_TYPE.Unknown, {}, None, None, None, self._settings) self._events.put(( self._listeners.publish_topology_description_changed, (initial_td, self._description, self._topology_id))) for seed in topology_settings.seeds: if self._publish_server: self._events.put((self._listeners.publish_server_opened, (seed, self._topology_id))) # Store the seed list to help diagnose errors in _error_message(). self._seed_addresses = list(topology_description.server_descriptions()) self._opened = False self._lock = threading.Lock() self._condition = self._settings.condition_class(self._lock) self._servers = {} self._pid = None self._max_cluster_time = None self._session_pool = _ServerSessionPool() if self._publish_server or self._publish_tp: def target(): return process_events_queue(weak) executor = periodic_executor.PeriodicExecutor( interval=common.EVENTS_QUEUE_FREQUENCY, min_interval=0.5, target=target, name="pymongo_events_thread") # We strongly reference the executor and it weakly references # the queue via this closure. When the topology is freed, stop # the executor soon. weak = weakref.ref(self._events) self.__events_executor = executor executor.open() def open(self): """Start monitoring, or restart after a fork. No effect if called multiple times. .. warning:: Topology is shared among multiple threads and is protected by mutual exclusion. Using Topology from a process other than the one that initialized it will emit a warning and may result in deadlock. To prevent this from happening, MongoClient must be created after any forking. """ if self._pid is None: self._pid = os.getpid() else: if os.getpid() != self._pid: warnings.warn( "MongoClient opened before fork. Create MongoClient only " "after forking. See PyMongo's documentation for details: " "http://api.mongodb.org/python/current/faq.html#" "is-pymongo-fork-safe") with self._lock: self._ensure_opened() def select_servers(self, selector, server_selection_timeout=None, address=None): """Return a list of Servers matching selector, or time out. :Parameters: - `selector`: function that takes a list of Servers and returns a subset of them. - `server_selection_timeout` (optional): maximum seconds to wait. If not provided, the default value common.SERVER_SELECTION_TIMEOUT is used. - `address`: optional server address to select. Calls self.open() if needed. Raises exc:`ServerSelectionTimeoutError` after `server_selection_timeout` if no matching servers are found. """ if server_selection_timeout is None: server_timeout = self._settings.server_selection_timeout else: server_timeout = server_selection_timeout with self._lock: server_descriptions = self._select_servers_loop( selector, server_timeout, address) return [self.get_server_by_address(sd.address) for sd in server_descriptions] def _select_servers_loop(self, selector, timeout, address): """select_servers() guts. Hold the lock when calling this.""" now = _time() end_time = now + timeout server_descriptions = self._description.apply_selector( selector, address) while not server_descriptions: # No suitable servers. if timeout == 0 or now > end_time: raise ServerSelectionTimeoutError( self._error_message(selector)) self._ensure_opened() self._request_check_all() # Release the lock and wait for the topology description to # change, or for a timeout. We won't miss any changes that # came after our most recent apply_selector call, since we've # held the lock until now. self._condition.wait(common.MIN_HEARTBEAT_INTERVAL) self._description.check_compatible() now = _time() server_descriptions = self._description.apply_selector( selector, address) self._description.check_compatible() return server_descriptions def select_server(self, selector, server_selection_timeout=None, address=None): """Like select_servers, but choose a random server if several match.""" return random.choice(self.select_servers(selector, server_selection_timeout, address)) def select_server_by_address(self, address, server_selection_timeout=None): """Return a Server for "address", reconnecting if necessary. If the server's type is not known, request an immediate check of all servers. Time out after "server_selection_timeout" if the server cannot be reached. :Parameters: - `address`: A (host, port) pair. - `server_selection_timeout` (optional): maximum seconds to wait. If not provided, the default value common.SERVER_SELECTION_TIMEOUT is used. Calls self.open() if needed. Raises exc:`ServerSelectionTimeoutError` after `server_selection_timeout` if no matching servers are found. """ return self.select_server(any_server_selector, server_selection_timeout, address) def _process_change(self, server_description): """Process a new ServerDescription on an opened topology. Hold the lock when calling this. """ td_old = self._description if self._publish_server: old_server_description = td_old._server_descriptions[ server_description.address] self._events.put(( self._listeners.publish_server_description_changed, (old_server_description, server_description, server_description.address, self._topology_id))) self._description = updated_topology_description( self._description, server_description) self._update_servers() self._receive_cluster_time_no_lock(server_description.cluster_time) if self._publish_tp: self._events.put(( self._listeners.publish_topology_description_changed, (td_old, self._description, self._topology_id))) # Wake waiters in select_servers(). self._condition.notify_all() def on_change(self, server_description): """Process a new ServerDescription after an ismaster call completes.""" # We do no I/O holding the lock. with self._lock: # Monitors may continue working on ismaster calls for some time # after a call to Topology.close, so this method may be called at # any time. Ensure the topology is open before processing the # change. # Any monitored server was definitely in the topology description # once. Check if it's still in the description or if some state- # change removed it. E.g., we got a host list from the primary # that didn't include this server. if (self._opened and self._description.has_server(server_description.address)): self._process_change(server_description) def get_server_by_address(self, address): """Get a Server or None. Returns the current version of the server immediately, even if it's Unknown or absent from the topology. Only use this in unittests. In driver code, use select_server_by_address, since then you're assured a recent view of the server's type and wire protocol version. """ return self._servers.get(address) def has_server(self, address): return address in self._servers def get_primary(self): """Return primary's address or None.""" # Implemented here in Topology instead of MongoClient, so it can lock. with self._lock: topology_type = self._description.topology_type if topology_type != TOPOLOGY_TYPE.ReplicaSetWithPrimary: return None return writable_server_selector(self._new_selection())[0].address def _get_replica_set_members(self, selector): """Return set of replica set member addresses.""" # Implemented here in Topology instead of MongoClient, so it can lock. with self._lock: topology_type = self._description.topology_type if topology_type not in (TOPOLOGY_TYPE.ReplicaSetWithPrimary, TOPOLOGY_TYPE.ReplicaSetNoPrimary): return set() return set([sd.address for sd in selector(self._new_selection())]) def get_secondaries(self): """Return set of secondary addresses.""" return self._get_replica_set_members(secondary_server_selector) def get_arbiters(self): """Return set of arbiter addresses.""" return self._get_replica_set_members(arbiter_server_selector) def max_cluster_time(self): """Return a document, the highest seen $clusterTime.""" return self._max_cluster_time def _receive_cluster_time_no_lock(self, cluster_time): # Driver Sessions Spec: "Whenever a driver receives a cluster time from # a server it MUST compare it to the current highest seen cluster time # for the deployment. If the new cluster time is higher than the # highest seen cluster time it MUST become the new highest seen cluster # time. Two cluster times are compared using only the BsonTimestamp # value of the clusterTime embedded field." if cluster_time: # ">" uses bson.timestamp.Timestamp's comparison operator. if (not self._max_cluster_time or cluster_time['clusterTime'] > self._max_cluster_time['clusterTime']): self._max_cluster_time = cluster_time def receive_cluster_time(self, cluster_time): with self._lock: self._receive_cluster_time_no_lock(cluster_time) def request_check_all(self, wait_time=5): """Wake all monitors, wait for at least one to check its server.""" with self._lock: self._request_check_all() self._condition.wait(wait_time) def reset_pool(self, address): with self._lock: server = self._servers.get(address) if server: server.pool.reset() def reset_server(self, address): """Clear our pool for a server and mark it Unknown. Do *not* request an immediate check. """ with self._lock: self._reset_server(address) def reset_server_and_request_check(self, address): """Clear our pool for a server, mark it Unknown, and check it soon.""" with self._lock: self._reset_server(address) self._request_check(address) def update_pool(self): # Remove any stale sockets and add new sockets if pool is too small. with self._lock: for server in self._servers.values(): server._pool.remove_stale_sockets() def close(self): """Clear pools and terminate monitors. Topology reopens on demand.""" with self._lock: for server in self._servers.values(): server.close() # Mark all servers Unknown. self._description = self._description.reset() self._update_servers() self._opened = False # Publish only after releasing the lock. if self._publish_tp: self._events.put((self._listeners.publish_topology_closed, (self._topology_id,))) if self._publish_server or self._publish_tp: self.__events_executor.close() @property def description(self): return self._description def pop_all_sessions(self): """Pop all session ids from the pool.""" with self._lock: return self._session_pool.pop_all() def get_server_session(self): """Start or resume a server session, or raise ConfigurationError.""" with self._lock: session_timeout = self._description.logical_session_timeout_minutes if session_timeout is None: # Maybe we need an initial scan? Can raise ServerSelectionError. if self._description.topology_type == TOPOLOGY_TYPE.Single: if not self._description.has_known_servers: self._select_servers_loop( any_server_selector, self._settings.server_selection_timeout, None) elif not self._description.readable_servers: self._select_servers_loop( readable_server_selector, self._settings.server_selection_timeout, None) session_timeout = self._description.logical_session_timeout_minutes if session_timeout is None: raise ConfigurationError( "Sessions are not supported by this MongoDB deployment") return self._session_pool.get_server_session(session_timeout) def return_server_session(self, server_session, lock): if lock: with self._lock: session_timeout = \ self._description.logical_session_timeout_minutes if session_timeout is not None: self._session_pool.return_server_session(server_session, session_timeout) else: # Called from a __del__ method, can't use a lock. self._session_pool.return_server_session_no_lock(server_session) def _new_selection(self): """A Selection object, initially including all known servers. Hold the lock when calling this. """ return Selection.from_topology_description(self._description) def _ensure_opened(self): """Start monitors, or restart after a fork. Hold the lock when calling this. """ if not self._opened: self._opened = True self._update_servers() # Start or restart the events publishing thread. if self._publish_tp or self._publish_server: self.__events_executor.open() # Ensure that the monitors are open. for server in itervalues(self._servers): server.open() def _reset_server(self, address): """Clear our pool for a server and mark it Unknown. Hold the lock when calling this. Does *not* request an immediate check. """ server = self._servers.get(address) # "server" is None if another thread removed it from the topology. if server: server.reset() # Mark this server Unknown. self._description = self._description.reset_server(address) self._update_servers() def _request_check(self, address): """Wake one monitor. Hold the lock when calling this.""" server = self._servers.get(address) # "server" is None if another thread removed it from the topology. if server: server.request_check() def _request_check_all(self): """Wake all monitors. Hold the lock when calling this.""" for server in self._servers.values(): server.request_check() def _update_servers(self): """Sync our Servers from TopologyDescription.server_descriptions. Hold the lock while calling this. """ for address, sd in self._description.server_descriptions().items(): if address not in self._servers: monitor = self._settings.monitor_class( server_description=sd, topology=self, pool=self._create_pool_for_monitor(address), topology_settings=self._settings) weak = None if self._publish_server: weak = weakref.ref(self._events) server = Server( server_description=sd, pool=self._create_pool_for_server(address), monitor=monitor, topology_id=self._topology_id, listeners=self._listeners, events=weak) self._servers[address] = server server.open() else: self._servers[address].description = sd for address, server in list(self._servers.items()): if not self._description.has_server(address): server.close() self._servers.pop(address) def _create_pool_for_server(self, address): return self._settings.pool_class(address, self._settings.pool_options) def _create_pool_for_monitor(self, address): options = self._settings.pool_options # According to the Server Discovery And Monitoring Spec, monitors use # connect_timeout for both connect_timeout and socket_timeout. The # pool only has one socket so maxPoolSize and so on aren't needed. monitor_pool_options = PoolOptions( connect_timeout=options.connect_timeout, socket_timeout=options.connect_timeout, ssl_context=options.ssl_context, ssl_match_hostname=options.ssl_match_hostname, event_listeners=options.event_listeners, appname=options.appname) return self._settings.pool_class(address, monitor_pool_options, handshake=False) def _error_message(self, selector): """Format an error message if server selection fails. Hold the lock when calling this. """ is_replica_set = self._description.topology_type in ( TOPOLOGY_TYPE.ReplicaSetWithPrimary, TOPOLOGY_TYPE.ReplicaSetNoPrimary) if is_replica_set: server_plural = 'replica set members' elif self._description.topology_type == TOPOLOGY_TYPE.Sharded: server_plural = 'mongoses' else: server_plural = 'servers' if self._description.known_servers: # We've connected, but no servers match the selector. if selector is writable_server_selector: if is_replica_set: return 'No primary available for writes' else: return 'No %s available for writes' % server_plural else: return 'No %s match selector "%s"' % (server_plural, selector) else: addresses = list(self._description.server_descriptions()) servers = list(self._description.server_descriptions().values()) if not servers: if is_replica_set: # We removed all servers because of the wrong setName? return 'No %s available for replica set name "%s"' % ( server_plural, self._settings.replica_set_name) else: return 'No %s available' % server_plural # 1 or more servers, all Unknown. Are they unknown for one reason? error = servers[0].error same = all(server.error == error for server in servers[1:]) if same: if error is None: # We're still discovering. return 'No %s found yet' % server_plural if (is_replica_set and not set(addresses).intersection(self._seed_addresses)): # We replaced our seeds with new hosts but can't reach any. return ( 'Could not reach any servers in %s. Replica set is' ' configured with internal hostnames or IPs?' % addresses) return str(error) else: return ','.join(str(server.error) for server in servers if server.error) pymongo-3.6.1/pymongo/collection.py0000644000076600000240000040441013245621354017640 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Collection level utilities for Mongo.""" import collections import datetime import warnings from bson.code import Code from bson.objectid import ObjectId from bson.py3compat import (_unicode, abc, integer_types, string_type) from bson.raw_bson import RawBSONDocument from bson.codec_options import CodecOptions from bson.son import SON from pymongo import (common, helpers, message) from pymongo.bulk import BulkOperationBuilder, _Bulk from pymongo.command_cursor import CommandCursor, RawBatchCommandCursor from pymongo.common import ORDERED_TYPES from pymongo.collation import validate_collation_or_none from pymongo.change_stream import ChangeStream from pymongo.cursor import Cursor, RawBatchCursor from pymongo.errors import (BulkWriteError, ConfigurationError, InvalidName, OperationFailure) from pymongo.helpers import (_check_write_command_response, _raise_last_error) from pymongo.message import _UNICODE_REPLACE_CODEC_OPTIONS from pymongo.operations import IndexModel from pymongo.read_preferences import ReadPreference from pymongo.results import (BulkWriteResult, DeleteResult, InsertOneResult, InsertManyResult, UpdateResult) from pymongo.write_concern import WriteConcern _NO_OBJ_ERROR = "No matching object found" _UJOIN = u"%s.%s" class ReturnDocument(object): """An enum used with :meth:`~pymongo.collection.Collection.find_one_and_replace` and :meth:`~pymongo.collection.Collection.find_one_and_update`. """ BEFORE = False """Return the original document before it was updated/replaced, or ``None`` if no document matches the query. """ AFTER = True """Return the updated/replaced or inserted document.""" class Collection(common.BaseObject): """A Mongo collection. """ def __init__(self, database, name, create=False, codec_options=None, read_preference=None, write_concern=None, read_concern=None, session=None, **kwargs): """Get / create a Mongo collection. Raises :class:`TypeError` if `name` is not an instance of :class:`basestring` (:class:`str` in python 3). Raises :class:`~pymongo.errors.InvalidName` if `name` is not a valid collection name. Any additional keyword arguments will be used as options passed to the create command. See :meth:`~pymongo.database.Database.create_collection` for valid options. If `create` is ``True``, `collation` is specified, or any additional keyword arguments are present, a ``create`` command will be sent, using ``session`` if specified. Otherwise, a ``create`` command will not be sent and the collection will be created implicitly on first use. The optional ``session`` argument is *only* used for the ``create`` command, it is not associated with the collection afterward. :Parameters: - `database`: the database to get a collection from - `name`: the name of the collection to get - `create` (optional): if ``True``, force collection creation even without options being set - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) database.codec_options is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) database.read_preference is used. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) database.write_concern is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) database.read_concern is used. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. If a collation is provided, it will be passed to the create collection command. This option is only supported on MongoDB 3.4 and above. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` that is used with the create collection command - `**kwargs` (optional): additional keyword arguments will be passed as options for the create collection command .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Support the `collation` option. .. versionchanged:: 3.2 Added the read_concern option. .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. Removed the uuid_subtype attribute. :class:`~pymongo.collection.Collection` no longer returns an instance of :class:`~pymongo.collection.Collection` for attribute names with leading underscores. You must use dict-style lookups instead:: collection['__my_collection__'] Not: collection.__my_collection__ .. versionchanged:: 2.2 Removed deprecated argument: options .. versionadded:: 2.1 uuid_subtype attribute .. mongodoc:: collections """ super(Collection, self).__init__( codec_options or database.codec_options, read_preference or database.read_preference, write_concern or database.write_concern, read_concern or database.read_concern) if not isinstance(name, string_type): raise TypeError("name must be an instance " "of %s" % (string_type.__name__,)) if not name or ".." in name: raise InvalidName("collection names cannot be empty") if "$" in name and not (name.startswith("oplog.$main") or name.startswith("$cmd")): raise InvalidName("collection names must not " "contain '$': %r" % name) if name[0] == "." or name[-1] == ".": raise InvalidName("collection names must not start " "or end with '.': %r" % name) if "\x00" in name: raise InvalidName("collection names must not contain the " "null character") collation = validate_collation_or_none(kwargs.pop('collation', None)) self.__database = database self.__name = _unicode(name) self.__full_name = _UJOIN % (self.__database.name, self.__name) if create or kwargs or collation: self.__create(kwargs, collation, session) self.__write_response_codec_options = self.codec_options._replace( unicode_decode_error_handler='replace', document_class=dict) def _socket_for_reads(self): return self.__database.client._socket_for_reads(self.read_preference) def _socket_for_primary_reads(self): return self.__database.client._socket_for_reads(ReadPreference.PRIMARY) def _socket_for_writes(self): return self.__database.client._socket_for_writes() def _command(self, sock_info, command, slave_ok=False, read_preference=None, codec_options=None, check=True, allowable_errors=None, read_concern=None, write_concern=None, parse_write_concern_error=False, collation=None, session=None, retryable_write=False): """Internal command helper. :Parameters: - `sock_info` - A SocketInfo instance. - `command` - The command itself, as a SON instance. - `slave_ok`: whether to set the SlaveOkay wire protocol bit. - `codec_options` (optional) - An instance of :class:`~bson.codec_options.CodecOptions`. - `check`: raise OperationFailure if there are errors - `allowable_errors`: errors to ignore if `check` is True - `read_concern` (optional) - An instance of :class:`~pymongo.read_concern.ReadConcern`. - `write_concern`: An instance of :class:`~pymongo.write_concern.WriteConcern`. This option is only valid for MongoDB 3.4 and above. - `parse_write_concern_error` (optional): Whether to parse a ``writeConcernError`` field in the command response. - `collation` (optional) - An instance of :class:`~pymongo.collation.Collation`. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: The result document. """ with self.__database.client._tmp_session(session) as s: return sock_info.command( self.__database.name, command, slave_ok, read_preference or self.read_preference, codec_options or self.codec_options, check, allowable_errors, read_concern=read_concern, write_concern=write_concern, parse_write_concern_error=parse_write_concern_error, collation=collation, session=s, client=self.__database.client, retryable_write=retryable_write) def __create(self, options, collation, session): """Sends a create command with the given options. """ cmd = SON([("create", self.__name)]) if options: if "size" in options: options["size"] = float(options["size"]) cmd.update(options) with self._socket_for_writes() as sock_info: self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY, write_concern=self.write_concern, parse_write_concern_error=True, collation=collation, session=session) def __getattr__(self, name): """Get a sub-collection of this collection by name. Raises InvalidName if an invalid collection name is used. :Parameters: - `name`: the name of the collection to get """ if name.startswith('_'): full_name = _UJOIN % (self.__name, name) raise AttributeError( "Collection has no attribute %r. To access the %s" " collection, use database['%s']." % ( name, full_name, full_name)) return self.__getitem__(name) def __getitem__(self, name): return Collection(self.__database, _UJOIN % (self.__name, name)) def __repr__(self): return "Collection(%r, %r)" % (self.__database, self.__name) def __eq__(self, other): if isinstance(other, Collection): return (self.__database == other.database and self.__name == other.name) return NotImplemented def __ne__(self, other): return not self == other @property def full_name(self): """The full name of this :class:`Collection`. The full name is of the form `database_name.collection_name`. """ return self.__full_name @property def name(self): """The name of this :class:`Collection`.""" return self.__name @property def database(self): """The :class:`~pymongo.database.Database` that this :class:`Collection` is a part of. """ return self.__database def with_options( self, codec_options=None, read_preference=None, write_concern=None, read_concern=None): """Get a clone of this collection changing the specified settings. >>> coll1.read_preference Primary() >>> from pymongo import ReadPreference >>> coll2 = coll1.with_options(read_preference=ReadPreference.SECONDARY) >>> coll1.read_preference Primary() >>> coll2.read_preference Secondary(tag_sets=None) :Parameters: - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`Collection` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`Collection` is used. See :mod:`~pymongo.read_preferences` for options. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`Collection` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`Collection` is used. """ return Collection(self.__database, self.__name, False, codec_options or self.codec_options, read_preference or self.read_preference, write_concern or self.write_concern, read_concern or self.read_concern) def initialize_unordered_bulk_op(self, bypass_document_validation=False): """**DEPRECATED** - Initialize an unordered batch of write operations. Operations will be performed on the server in arbitrary order, possibly in parallel. All operations will be attempted. :Parameters: - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. Returns a :class:`~pymongo.bulk.BulkOperationBuilder` instance. See :ref:`unordered_bulk` for examples. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.5 Deprecated. Use :meth:`~pymongo.collection.Collection.bulk_write` instead. .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 2.7 """ warnings.warn("initialize_unordered_bulk_op is deprecated", DeprecationWarning, stacklevel=2) return BulkOperationBuilder(self, False, bypass_document_validation) def initialize_ordered_bulk_op(self, bypass_document_validation=False): """**DEPRECATED** - Initialize an ordered batch of write operations. Operations will be performed on the server serially, in the order provided. If an error occurs all remaining operations are aborted. :Parameters: - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. Returns a :class:`~pymongo.bulk.BulkOperationBuilder` instance. See :ref:`ordered_bulk` for examples. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.5 Deprecated. Use :meth:`~pymongo.collection.Collection.bulk_write` instead. .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 2.7 """ warnings.warn("initialize_ordered_bulk_op is deprecated", DeprecationWarning, stacklevel=2) return BulkOperationBuilder(self, True, bypass_document_validation) def bulk_write(self, requests, ordered=True, bypass_document_validation=False, session=None): """Send a batch of write operations to the server. Requests are passed as a list of write operation instances ( :class:`~pymongo.operations.InsertOne`, :class:`~pymongo.operations.UpdateOne`, :class:`~pymongo.operations.UpdateMany`, :class:`~pymongo.operations.ReplaceOne`, :class:`~pymongo.operations.DeleteOne`, or :class:`~pymongo.operations.DeleteMany`). >>> for doc in db.test.find({}): ... print(doc) ... {u'x': 1, u'_id': ObjectId('54f62e60fba5226811f634ef')} {u'x': 1, u'_id': ObjectId('54f62e60fba5226811f634f0')} >>> # DeleteMany, UpdateOne, and UpdateMany are also available. ... >>> from pymongo import InsertOne, DeleteOne, ReplaceOne >>> requests = [InsertOne({'y': 1}), DeleteOne({'x': 1}), ... ReplaceOne({'w': 1}, {'z': 1}, upsert=True)] >>> result = db.test.bulk_write(requests) >>> result.inserted_count 1 >>> result.deleted_count 1 >>> result.modified_count 0 >>> result.upserted_ids {2: ObjectId('54f62ee28891e756a6e1abd5')} >>> for doc in db.test.find({}): ... print(doc) ... {u'x': 1, u'_id': ObjectId('54f62e60fba5226811f634f0')} {u'y': 1, u'_id': ObjectId('54f62ee2fba5226811f634f1')} {u'z': 1, u'_id': ObjectId('54f62ee28891e756a6e1abd5')} :Parameters: - `requests`: A list of write operations (see examples above). - `ordered` (optional): If ``True`` (the default) requests will be performed on the server serially, in the order provided. If an error occurs all remaining operations are aborted. If ``False`` requests will be performed on the server in arbitrary order, possibly in parallel, and all operations will be attempted. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: An instance of :class:`~pymongo.results.BulkWriteResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_list("requests", requests) blk = _Bulk(self, ordered, bypass_document_validation) for request in requests: try: request._add_to_bulk(blk) except AttributeError: raise TypeError("%r is not a valid request" % (request,)) bulk_api_result = blk.execute(self.write_concern.document, session) if bulk_api_result is not None: return BulkWriteResult(bulk_api_result, True) return BulkWriteResult({}, False) def _legacy_write(self, sock_info, name, cmd, op_id, bypass_doc_val, func, *args): """Internal legacy unacknowledged write helper.""" # Cannot have both unacknowledged write and bypass document validation. if bypass_doc_val and sock_info.max_wire_version >= 4: raise OperationFailure("Cannot set bypass_document_validation with" " unacknowledged write concern") listeners = self.database.client._event_listeners publish = listeners.enabled_for_commands if publish: start = datetime.datetime.now() rqst_id, msg, max_size = func(*args) if publish: duration = datetime.datetime.now() - start listeners.publish_command_start( cmd, self.__database.name, rqst_id, sock_info.address, op_id) start = datetime.datetime.now() try: result = sock_info.legacy_write(rqst_id, msg, max_size, False) except Exception as exc: if publish: dur = (datetime.datetime.now() - start) + duration if isinstance(exc, OperationFailure): details = exc.details # Succeed if GLE was successful and this is a write error. if details.get("ok") and "n" in details: reply = message._convert_write_result( name, cmd, details) listeners.publish_command_success( dur, reply, name, rqst_id, sock_info.address, op_id) raise else: details = message._convert_exception(exc) listeners.publish_command_failure( dur, details, name, rqst_id, sock_info.address, op_id) raise if publish: if result is not None: reply = message._convert_write_result(name, cmd, result) else: # Comply with APM spec. reply = {'ok': 1} duration = (datetime.datetime.now() - start) + duration listeners.publish_command_success( duration, reply, name, rqst_id, sock_info.address, op_id) return result def _insert_one( self, doc, ordered, check_keys, manipulate, write_concern, op_id, bypass_doc_val, session): """Internal helper for inserting a single document.""" if manipulate: doc = self.__database._apply_incoming_manipulators(doc, self) if not isinstance(doc, RawBSONDocument) and '_id' not in doc: doc['_id'] = ObjectId() doc = self.__database._apply_incoming_copying_manipulators(doc, self) concern = (write_concern or self.write_concern).document acknowledged = concern.get("w") != 0 command = SON([('insert', self.name), ('ordered', ordered), ('documents', [doc])]) if concern: command['writeConcern'] = concern if acknowledged: def _insert_command(session, sock_info, retryable_write): if bypass_doc_val and sock_info.max_wire_version >= 4: command['bypassDocumentValidation'] = True return sock_info.command( self.__database.name, command, codec_options=self.__write_response_codec_options, check_keys=check_keys, session=session, client=self.__database.client, retryable_write=retryable_write) result = self.__database.client._retryable_write( True, _insert_command, session) _check_write_command_response(result) else: with self._socket_for_writes() as sock_info: # Legacy OP_INSERT. self._legacy_write( sock_info, 'insert', command, op_id, bypass_doc_val, message.insert, self.__full_name, [doc], check_keys, False, concern, False, self.__write_response_codec_options) if not isinstance(doc, RawBSONDocument): return doc.get('_id') def _insert(self, docs, ordered=True, check_keys=True, manipulate=False, write_concern=None, op_id=None, bypass_doc_val=False, session=None): """Internal insert helper.""" if isinstance(docs, abc.Mapping): return self._insert_one( docs, ordered, check_keys, manipulate, write_concern, op_id, bypass_doc_val, session) ids = [] if manipulate: def gen(): """Generator that applies SON manipulators to each document and adds _id if necessary. """ _db = self.__database for doc in docs: # Apply user-configured SON manipulators. This order of # operations is required for backwards compatibility, # see PYTHON-709. doc = _db._apply_incoming_manipulators(doc, self) if not (isinstance(doc, RawBSONDocument) or '_id' in doc): doc['_id'] = ObjectId() doc = _db._apply_incoming_copying_manipulators(doc, self) ids.append(doc['_id']) yield doc else: def gen(): """Generator that only tracks existing _ids.""" for doc in docs: # Don't inflate RawBSONDocument by touching fields. if not isinstance(doc, RawBSONDocument): ids.append(doc.get('_id')) yield doc concern = (write_concern or self.write_concern).document blk = _Bulk(self, ordered, bypass_doc_val) blk.ops = [(message._INSERT, doc) for doc in gen()] try: blk.execute(concern, session=session) except BulkWriteError as bwe: _raise_last_error(bwe.details) return ids def insert_one(self, document, bypass_document_validation=False, session=None): """Insert a single document. >>> db.test.count({'x': 1}) 0 >>> result = db.test.insert_one({'x': 1}) >>> result.inserted_id ObjectId('54f112defba522406c9cc208') >>> db.test.find_one({'x': 1}) {u'x': 1, u'_id': ObjectId('54f112defba522406c9cc208')} :Parameters: - `document`: The document to insert. Must be a mutable mapping type. If the document does not have an _id field one will be added automatically. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: - An instance of :class:`~pymongo.results.InsertOneResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_is_document_type("document", document) if not (isinstance(document, RawBSONDocument) or "_id" in document): document["_id"] = ObjectId() return InsertOneResult( self._insert(document, bypass_doc_val=bypass_document_validation, session=session), self.write_concern.acknowledged) def insert_many(self, documents, ordered=True, bypass_document_validation=False, session=None): """Insert an iterable of documents. >>> db.test.count() 0 >>> result = db.test.insert_many([{'x': i} for i in range(2)]) >>> result.inserted_ids [ObjectId('54f113fffba522406c9cc20e'), ObjectId('54f113fffba522406c9cc20f')] >>> db.test.count() 2 :Parameters: - `documents`: A iterable of documents to insert. - `ordered` (optional): If ``True`` (the default) documents will be inserted on the server serially, in the order provided. If an error occurs all remaining inserts are aborted. If ``False``, documents will be inserted on the server in arbitrary order, possibly in parallel, and all document inserts will be attempted. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: An instance of :class:`~pymongo.results.InsertManyResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ if not isinstance(documents, abc.Iterable) or not documents: raise TypeError("documents must be a non-empty list") inserted_ids = [] def gen(): """A generator that validates documents and handles _ids.""" for document in documents: common.validate_is_document_type("document", document) if not isinstance(document, RawBSONDocument): if "_id" not in document: document["_id"] = ObjectId() inserted_ids.append(document["_id"]) yield (message._INSERT, document) blk = _Bulk(self, ordered, bypass_document_validation) blk.ops = [doc for doc in gen()] blk.execute(self.write_concern.document, session=session) return InsertManyResult(inserted_ids, self.write_concern.acknowledged) def _update(self, sock_info, criteria, document, upsert=False, check_keys=True, multi=False, manipulate=False, write_concern=None, op_id=None, ordered=True, bypass_doc_val=False, collation=None, array_filters=None, session=None, retryable_write=False): """Internal update / replace helper.""" common.validate_boolean("upsert", upsert) if manipulate: document = self.__database._fix_incoming(document, self) collation = validate_collation_or_none(collation) concern = (write_concern or self.write_concern).document acknowledged = concern.get("w") != 0 update_doc = SON([('q', criteria), ('u', document), ('multi', multi), ('upsert', upsert)]) if collation is not None: if sock_info.max_wire_version < 5: raise ConfigurationError( 'Must be connected to MongoDB 3.4+ to use collations.') elif not acknowledged: raise ConfigurationError( 'Collation is unsupported for unacknowledged writes.') else: update_doc['collation'] = collation if array_filters is not None: if sock_info.max_wire_version < 6: raise ConfigurationError( 'Must be connected to MongoDB 3.6+ to use array_filters.') elif not acknowledged: raise ConfigurationError( 'arrayFilters is unsupported for unacknowledged writes.') else: update_doc['arrayFilters'] = array_filters command = SON([('update', self.name), ('ordered', ordered), ('updates', [update_doc])]) if concern: command['writeConcern'] = concern if acknowledged: # Update command. if bypass_doc_val and sock_info.max_wire_version >= 4: command['bypassDocumentValidation'] = True # The command result has to be published for APM unmodified # so we make a shallow copy here before adding updatedExisting. result = sock_info.command( self.__database.name, command, codec_options=self.__write_response_codec_options, session=session, client=self.__database.client, retryable_write=retryable_write).copy() _check_write_command_response(result) # Add the updatedExisting field for compatibility. if result.get('n') and 'upserted' not in result: result['updatedExisting'] = True else: result['updatedExisting'] = False # MongoDB >= 2.6.0 returns the upsert _id in an array # element. Break it out for backward compatibility. if 'upserted' in result: result['upserted'] = result['upserted'][0]['_id'] return result else: # Legacy OP_UPDATE. return self._legacy_write( sock_info, 'update', command, op_id, bypass_doc_val, message.update, self.__full_name, upsert, multi, criteria, document, False, concern, check_keys, self.__write_response_codec_options) def _update_retryable( self, criteria, document, upsert=False, check_keys=True, multi=False, manipulate=False, write_concern=None, op_id=None, ordered=True, bypass_doc_val=False, collation=None, array_filters=None, session=None): """Internal update / replace helper.""" def _update(session, sock_info, retryable_write): return self._update( sock_info, criteria, document, upsert=upsert, check_keys=check_keys, multi=multi, manipulate=manipulate, write_concern=write_concern, op_id=op_id, ordered=ordered, bypass_doc_val=bypass_doc_val, collation=collation, array_filters=array_filters, session=session, retryable_write=retryable_write) return self.__database.client._retryable_write( (write_concern or self.write_concern).acknowledged and not multi, _update, session) def replace_one(self, filter, replacement, upsert=False, bypass_document_validation=False, collation=None, session=None): """Replace a single document matching the filter. >>> for doc in db.test.find({}): ... print(doc) ... {u'x': 1, u'_id': ObjectId('54f4c5befba5220aa4d6dee7')} >>> result = db.test.replace_one({'x': 1}, {'y': 1}) >>> result.matched_count 1 >>> result.modified_count 1 >>> for doc in db.test.find({}): ... print(doc) ... {u'y': 1, u'_id': ObjectId('54f4c5befba5220aa4d6dee7')} The *upsert* option can be used to insert a new document if a matching document does not exist. >>> result = db.test.replace_one({'x': 1}, {'x': 1}, True) >>> result.matched_count 0 >>> result.modified_count 0 >>> result.upserted_id ObjectId('54f11e5c8891e756a6e1abd4') >>> db.test.find_one({'x': 1}) {u'x': 1, u'_id': ObjectId('54f11e5c8891e756a6e1abd4')} :Parameters: - `filter`: A query that matches the document to replace. - `replacement`: The new document. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Added the `collation` option. .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_is_mapping("filter", filter) common.validate_ok_for_replace(replacement) return UpdateResult( self._update_retryable( filter, replacement, upsert, bypass_doc_val=bypass_document_validation, collation=collation, session=session), self.write_concern.acknowledged) def update_one(self, filter, update, upsert=False, bypass_document_validation=False, collation=None, array_filters=None, session=None): """Update a single document matching the filter. >>> for doc in db.test.find(): ... print(doc) ... {u'x': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} >>> result = db.test.update_one({'x': 1}, {'$inc': {'x': 3}}) >>> result.matched_count 1 >>> result.modified_count 1 >>> for doc in db.test.find(): ... print(doc) ... {u'x': 4, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} :Parameters: - `filter`: A query that matches the document to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `array_filters` (optional): A list of filters specifying which array elements an update should apply. Requires MongoDB 3.6+. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.6 Added the `array_filters` and ``session`` parameters. .. versionchanged:: 3.4 Added the `collation` option. .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_is_mapping("filter", filter) common.validate_ok_for_update(update) common.validate_list_or_none('array_filters', array_filters) return UpdateResult( self._update_retryable( filter, update, upsert, check_keys=False, bypass_doc_val=bypass_document_validation, collation=collation, array_filters=array_filters, session=session), self.write_concern.acknowledged) def update_many(self, filter, update, upsert=False, array_filters=None, bypass_document_validation=False, collation=None, session=None): """Update one or more documents that match the filter. >>> for doc in db.test.find(): ... print(doc) ... {u'x': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} >>> result = db.test.update_many({'x': 1}, {'$inc': {'x': 3}}) >>> result.matched_count 3 >>> result.modified_count 3 >>> for doc in db.test.find(): ... print(doc) ... {u'x': 4, u'_id': 0} {u'x': 4, u'_id': 1} {u'x': 4, u'_id': 2} :Parameters: - `filter`: A query that matches the documents to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation` (optional): If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `array_filters` (optional): A list of filters specifying which array elements an update should apply. Requires MongoDB 3.6+. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.6 Added ``array_filters`` and ``session`` parameters. .. versionchanged:: 3.4 Added the `collation` option. .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_is_mapping("filter", filter) common.validate_ok_for_update(update) common.validate_list_or_none('array_filters', array_filters) return UpdateResult( self._update_retryable( filter, update, upsert, check_keys=False, multi=True, bypass_doc_val=bypass_document_validation, collation=collation, array_filters=array_filters, session=session), self.write_concern.acknowledged) def drop(self, session=None): """Alias for :meth:`~pymongo.database.Database.drop_collection`. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. The following two calls are equivalent: >>> db.foo.drop() >>> db.drop_collection("foo") .. versionchanged:: 3.6 Added ``session`` parameter. """ self.__database.drop_collection(self.__name, session=session) def _delete( self, sock_info, criteria, multi, write_concern=None, op_id=None, ordered=True, collation=None, session=None, retryable_write=False): """Internal delete helper.""" common.validate_is_mapping("filter", criteria) concern = (write_concern or self.write_concern).document acknowledged = concern.get("w") != 0 delete_doc = SON([('q', criteria), ('limit', int(not multi))]) collation = validate_collation_or_none(collation) if collation is not None: if sock_info.max_wire_version < 5: raise ConfigurationError( 'Must be connected to MongoDB 3.4+ to use collations.') elif not acknowledged: raise ConfigurationError( 'Collation is unsupported for unacknowledged writes.') else: delete_doc['collation'] = collation command = SON([('delete', self.name), ('ordered', ordered), ('deletes', [delete_doc])]) if concern: command['writeConcern'] = concern if acknowledged: # Delete command. result = sock_info.command( self.__database.name, command, codec_options=self.__write_response_codec_options, session=session, client=self.__database.client, retryable_write=retryable_write) _check_write_command_response(result) return result else: # Legacy OP_DELETE. return self._legacy_write( sock_info, 'delete', command, op_id, False, message.delete, self.__full_name, criteria, False, concern, self.__write_response_codec_options, int(not multi)) def _delete_retryable( self, criteria, multi, write_concern=None, op_id=None, ordered=True, collation=None, session=None): """Internal delete helper.""" def _delete(session, sock_info, retryable_write): return self._delete( sock_info, criteria, multi, write_concern=write_concern, op_id=op_id, ordered=ordered, collation=collation, session=session, retryable_write=retryable_write) return self.__database.client._retryable_write( (write_concern or self.write_concern).acknowledged and not multi, _delete, session) def delete_one(self, filter, collation=None, session=None): """Delete a single document matching the filter. >>> db.test.count({'x': 1}) 3 >>> result = db.test.delete_one({'x': 1}) >>> result.deleted_count 1 >>> db.test.count({'x': 1}) 2 :Parameters: - `filter`: A query that matches the document to delete. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: - An instance of :class:`~pymongo.results.DeleteResult`. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Added the `collation` option. .. versionadded:: 3.0 """ return DeleteResult( self._delete_retryable( filter, False, collation=collation, session=session), self.write_concern.acknowledged) def delete_many(self, filter, collation=None, session=None): """Delete one or more documents matching the filter. >>> db.test.count({'x': 1}) 3 >>> result = db.test.delete_many({'x': 1}) >>> result.deleted_count 3 >>> db.test.count({'x': 1}) 0 :Parameters: - `filter`: A query that matches the documents to delete. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: - An instance of :class:`~pymongo.results.DeleteResult`. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Added the `collation` option. .. versionadded:: 3.0 """ return DeleteResult( self._delete_retryable( filter, True, collation=collation, session=session), self.write_concern.acknowledged) def find_one(self, filter=None, *args, **kwargs): """Get a single document from the database. All arguments to :meth:`find` are also valid arguments for :meth:`find_one`, although any `limit` argument will be ignored. Returns a single document, or ``None`` if no matching document is found. The :meth:`find_one` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `filter` (optional): a dictionary specifying the query to be performed OR any other type to be used as the value for a query for ``"_id"``. - `*args` (optional): any additional positional arguments are the same as the arguments to :meth:`find`. - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`find`. >>> collection.find_one(max_time_ms=100) """ if (filter is not None and not isinstance(filter, abc.Mapping)): filter = {"_id": filter} cursor = self.find(filter, *args, **kwargs) for result in cursor.limit(-1): return result return None def find(self, *args, **kwargs): """Query the database. The `filter` argument is a prototype document that all results must match. For example: >>> db.test.find({"hello": "world"}) only matches documents that have a key "hello" with value "world". Matches can have other keys *in addition* to "hello". The `projection` argument is used to specify a subset of fields that should be included in the result documents. By limiting results to a certain subset of fields you can cut down on network traffic and decoding time. Raises :class:`TypeError` if any of the arguments are of improper type. Returns an instance of :class:`~pymongo.cursor.Cursor` corresponding to this query. The :meth:`find` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `filter` (optional): a SON object specifying elements which must be present for a document to be included in the result set - `projection` (optional): a list of field names that should be returned in the result set or a dict specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a dict to exclude fields from the result (e.g. projection={'_id': False}). - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `skip` (optional): the number of documents to omit (from the start of the result set) when returning the results - `limit` (optional): the maximum number of results to return - `no_cursor_timeout` (optional): if False (the default), any returned cursor is closed by the server after 10 minutes of inactivity. If set to True, the returned cursor will never time out on the server. Care should be taken to ensure that cursors with no_cursor_timeout turned on are properly closed. - `cursor_type` (optional): the type of cursor to return. The valid options are defined by :class:`~pymongo.cursor.CursorType`: - :attr:`~pymongo.cursor.CursorType.NON_TAILABLE` - the result of this find call will return a standard cursor over the result set. - :attr:`~pymongo.cursor.CursorType.TAILABLE` - the result of this find call will be a tailable cursor - tailable cursors are only for use with capped collections. They are not closed when the last data is retrieved but are kept open and the cursor location marks the final document position. If more data is received iteration of the cursor will continue from the last document received. For details, see the `tailable cursor documentation `_. - :attr:`~pymongo.cursor.CursorType.TAILABLE_AWAIT` - the result of this find call will be a tailable cursor with the await flag set. The server will wait for a few seconds after returning the full result set so that it can capture and return additional data added during the query. - :attr:`~pymongo.cursor.CursorType.EXHAUST` - the result of this find call will be an exhaust cursor. MongoDB will stream batched results to the client without waiting for the client to request each batch, reducing latency. See notes on compatibility below. - `sort` (optional): a list of (key, direction) pairs specifying the sort order for this query. See :meth:`~pymongo.cursor.Cursor.sort` for details. - `allow_partial_results` (optional): if True, mongos will return partial results if some shards are down instead of returning an error. - `oplog_replay` (optional): If True, set the oplogReplay query flag. - `batch_size` (optional): Limits the number of documents returned in a single batch. - `manipulate` (optional): **DEPRECATED** - If True (the default), apply any outgoing SON manipulators before returning. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `return_key` (optional): If True, return only the index keys in each document. - `show_record_id` (optional): If True, adds a field ``$recordId`` in each document with the storage engine's internal record identifier. - `snapshot` (optional): If True, prevents the cursor from returning a document more than once because of an intervening write operation. - `hint` (optional): An index, in the same format as passed to :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``). Pass this as an alternative to calling :meth:`~pymongo.cursor.Cursor.hint` on the cursor to tell Mongo the proper index to use for the query. - `max_time_ms` (optional): Specifies a time limit for a query operation. If the specified time is exceeded, the operation will be aborted and :exc:`~pymongo.errors.ExecutionTimeout` is raised. Pass this as an alternative to calling :meth:`~pymongo.cursor.Cursor.max_time_ms` on the cursor. - `max_scan` (optional): The maximum number of documents to scan. Pass this as an alternative to calling :meth:`~pymongo.cursor.Cursor.max_scan` on the cursor. - `min` (optional): A list of field, limit pairs specifying the inclusive lower bound for all keys of a specific index in order. Pass this as an alternative to calling :meth:`~pymongo.cursor.Cursor.min` on the cursor. - `max` (optional): A list of field, limit pairs specifying the exclusive upper bound for all keys of a specific index in order. Pass this as an alternative to calling :meth:`~pymongo.cursor.Cursor.max` on the cursor. - `comment` (optional): A string or document. Pass this as an alternative to calling :meth:`~pymongo.cursor.Cursor.comment` on the cursor. - `modifiers` (optional): **DEPRECATED** - A dict specifying additional MongoDB query modifiers. Use the keyword arguments listed above instead. .. note:: There are a number of caveats to using :attr:`~pymongo.cursor.CursorType.EXHAUST` as cursor_type: - The `limit` option can not be used with an exhaust cursor. - Exhaust cursors are not supported by mongos and can not be used with a sharded cluster. - A :class:`~pymongo.cursor.Cursor` instance created with the :attr:`~pymongo.cursor.CursorType.EXHAUST` cursor_type requires an exclusive :class:`~socket.socket` connection to MongoDB. If the :class:`~pymongo.cursor.Cursor` is discarded without being completely iterated the underlying :class:`~socket.socket` connection will be closed and discarded without being returned to the connection pool. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.5 Added the options `return_key`, `show_record_id`, `snapshot`, `hint`, `max_time_ms`, `max_scan`, `min`, `max`, and `comment`. Deprecated the option `modifiers`. .. versionchanged:: 3.4 Support the `collation` option. .. versionchanged:: 3.0 Changed the parameter names `spec`, `fields`, `timeout`, and `partial` to `filter`, `projection`, `no_cursor_timeout`, and `allow_partial_results` respectively. Added the `cursor_type`, `oplog_replay`, and `modifiers` options. Removed the `network_timeout`, `read_preference`, `tag_sets`, `secondary_acceptable_latency_ms`, `max_scan`, `snapshot`, `tailable`, `await_data`, `exhaust`, `as_class`, and slave_okay parameters. Removed `compile_re` option: PyMongo now always represents BSON regular expressions as :class:`~bson.regex.Regex` objects. Use :meth:`~bson.regex.Regex.try_compile` to attempt to convert from a BSON regular expression to a Python regular expression object. Soft deprecated the `manipulate` option. .. versionchanged:: 2.7 Added `compile_re` option. If set to False, PyMongo represented BSON regular expressions as :class:`~bson.regex.Regex` objects instead of attempting to compile BSON regular expressions as Python native regular expressions, thus preventing errors for some incompatible patterns, see `PYTHON-500`_. .. versionadded:: 2.3 The `tag_sets` and `secondary_acceptable_latency_ms` parameters. .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500 .. mongodoc:: find """ return Cursor(self, *args, **kwargs) def find_raw_batches(self, *args, **kwargs): """Query the database and retrieve batches of raw BSON. Similar to the :meth:`find` method but returns a :class:`~pymongo.cursor.RawBatchCursor`. This example demonstrates how to work with raw batches, but in practice raw batches should be passed to an external library that can decode BSON into another data type, rather than used with PyMongo's :mod:`bson` module. >>> import bson >>> cursor = db.test.find_raw_batches() >>> for batch in cursor: ... print(bson.decode_all(batch)) .. note:: find_raw_batches does not support sessions. .. versionadded:: 3.6 """ # OP_MSG with document stream returns is required to support # sessions. if "session" in kwargs: raise ConfigurationError( "find_raw_batches does not support sessions") return RawBatchCursor(self, *args, **kwargs) def parallel_scan(self, num_cursors, session=None, **kwargs): """Scan this entire collection in parallel. Returns a list of up to ``num_cursors`` cursors that can be iterated concurrently. As long as the collection is not modified during scanning, each document appears once in one of the cursors result sets. For example, to process each document in a collection using some thread-safe ``process_document()`` function: >>> def process_cursor(cursor): ... for document in cursor: ... # Some thread-safe processing function: ... process_document(document) >>> >>> # Get up to 4 cursors. ... >>> cursors = collection.parallel_scan(4) >>> threads = [ ... threading.Thread(target=process_cursor, args=(cursor,)) ... for cursor in cursors] >>> >>> for thread in threads: ... thread.start() >>> >>> for thread in threads: ... thread.join() >>> >>> # All documents have now been processed. The :meth:`parallel_scan` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `num_cursors`: the number of cursors to return - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs`: additional options for the parallelCollectionScan command can be passed as keyword arguments. .. note:: Requires server version **>= 2.5.5**. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Added back support for arbitrary keyword arguments. MongoDB 3.4 adds support for maxTimeMS as an option to the parallelCollectionScan command. .. versionchanged:: 3.0 Removed support for arbitrary keyword arguments, since the parallelCollectionScan command has no optional arguments. """ cmd = SON([('parallelCollectionScan', self.__name), ('numCursors', num_cursors)]) cmd.update(kwargs) with self._socket_for_reads() as (sock_info, slave_ok): result = self._command(sock_info, cmd, slave_ok, read_concern=self.read_concern, session=session) cursors = [] for cursor in result['cursors']: s = self.__database.client._ensure_session(session) cursors.append(CommandCursor( self, cursor['cursor'], sock_info.address, session=s, explicit_session=session is not None)) return cursors def _count(self, cmd, collation=None, session=None): """Internal count helper.""" with self._socket_for_reads() as (sock_info, slave_ok): res = self._command( sock_info, cmd, slave_ok, allowable_errors=["ns missing"], codec_options=self.__write_response_codec_options, read_concern=self.read_concern, collation=collation, session=session) if res.get("errmsg", "") == "ns missing": return 0 return int(res["n"]) def count(self, filter=None, session=None, **kwargs): """Get the number of documents in this collection. All optional count parameters should be passed as keyword arguments to this method. Valid options include: - `hint` (string or list of tuples): The index to use. Specify either the index name as a string or the index specification as a list of tuples (e.g. [('a', pymongo.ASCENDING), ('b', pymongo.ASCENDING)]). - `limit` (int): The maximum number of documents to count. - `skip` (int): The number of matching documents to skip before returning results. - `maxTimeMS` (int): The maximum amount of time to allow the count command to run, in milliseconds. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. The :meth:`count` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `filter` (optional): A query document that selects which documents to count in the collection. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): See list of options above. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Support the `collation` option. """ cmd = SON([("count", self.__name)]) if filter is not None: if "query" in kwargs: raise ConfigurationError("can't pass both filter and query") kwargs["query"] = filter if "hint" in kwargs and not isinstance(kwargs["hint"], string_type): kwargs["hint"] = helpers._index_document(kwargs["hint"]) collation = validate_collation_or_none(kwargs.pop('collation', None)) cmd.update(kwargs) return self._count(cmd, collation, session) def create_indexes(self, indexes, session=None, **kwargs): """Create one or more indexes on this collection. >>> from pymongo import IndexModel, ASCENDING, DESCENDING >>> index1 = IndexModel([("hello", DESCENDING), ... ("world", ASCENDING)], name="hello_world") >>> index2 = IndexModel([("goodbye", DESCENDING)]) >>> db.test.create_indexes([index1, index2]) ["hello_world", "goodbye_-1"] :Parameters: - `indexes`: A list of :class:`~pymongo.operations.IndexModel` instances. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): optional arguments to the createIndexes command (like maxTimeMS) can be passed as keyword arguments. .. note:: `create_indexes` uses the `createIndexes`_ command introduced in MongoDB **2.6** and cannot be used with earlier versions. .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. .. versionchanged:: 3.6 Added ``session`` parameter. Added support for arbitrary keyword arguments. .. versionchanged:: 3.4 Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. .. versionadded:: 3.0 .. _createIndexes: https://docs.mongodb.com/manual/reference/command/createIndexes/ """ common.validate_list('indexes', indexes) names = [] with self._socket_for_writes() as sock_info: supports_collations = sock_info.max_wire_version >= 5 def gen_indexes(): for index in indexes: if not isinstance(index, IndexModel): raise TypeError( "%r is not an instance of " "pymongo.operations.IndexModel" % (index,)) document = index.document if "collation" in document and not supports_collations: raise ConfigurationError( "Must be connected to MongoDB " "3.4+ to use collations.") names.append(document["name"]) yield document cmd = SON([('createIndexes', self.name), ('indexes', list(gen_indexes()))]) cmd.update(kwargs) self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY, codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, write_concern=self.write_concern, parse_write_concern_error=True, session=session) return names def __create_index(self, keys, index_options, session, **kwargs): """Internal create index helper. :Parameters: - `keys`: a list of tuples [(key, type), (key, type), ...] - `index_options`: a dict of index options. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. """ index_doc = helpers._index_document(keys) index = {"key": index_doc} collation = validate_collation_or_none( index_options.pop('collation', None)) index.update(index_options) with self._socket_for_writes() as sock_info: if collation is not None: if sock_info.max_wire_version < 5: raise ConfigurationError( 'Must be connected to MongoDB 3.4+ to use collations.') else: index['collation'] = collation cmd = SON([('createIndexes', self.name), ('indexes', [index])]) cmd.update(kwargs) self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY, codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, write_concern=self.write_concern, parse_write_concern_error=True, session=session) def create_index(self, keys, session=None, **kwargs): """Creates an index on this collection. Takes either a single key or a list of (key, direction) pairs. The key(s) must be an instance of :class:`basestring` (:class:`str` in python 3), and the direction(s) must be one of (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`, :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`, :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`, :data:`~pymongo.TEXT`). To create a single key ascending index on the key ``'mike'`` we just use a string argument:: >>> my_collection.create_index("mike") For a compound index on ``'mike'`` descending and ``'eliot'`` ascending we need to use a list of tuples:: >>> my_collection.create_index([("mike", pymongo.DESCENDING), ... ("eliot", pymongo.ASCENDING)]) All optional index creation parameters should be passed as keyword arguments to this method. For example:: >>> my_collection.create_index([("mike", pymongo.DESCENDING)], ... background=True) Valid options include, but are not limited to: - `name`: custom name to use for this index - if none is given, a name will be generated. - `unique`: if ``True`` creates a uniqueness constraint on the index. - `background`: if ``True`` this index should be created in the background. - `sparse`: if ``True``, omit from the index any documents that lack the indexed field. - `bucketSize`: for use with geoHaystack indexes. Number of documents to group together within a certain proximity to a given longitude and latitude. - `min`: minimum value for keys in a :data:`~pymongo.GEO2D` index. - `max`: maximum value for keys in a :data:`~pymongo.GEO2D` index. - `expireAfterSeconds`: Used to create an expiring (TTL) collection. MongoDB will automatically delete documents from this collection after seconds. The indexed field must be a UTC datetime or the data will not expire. - `partialFilterExpression`: A document that specifies a filter for a partial index. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. See the MongoDB documentation for a full list of supported options by server version. .. warning:: `dropDups` is not supported by MongoDB 3.0 or newer. The option is silently ignored by the server and unique index builds using the option will fail if a duplicate value is detected. .. note:: `partialFilterExpression` requires server version **>= 3.2** .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. :Parameters: - `keys`: a single key or a list of (key, direction) pairs specifying the index to create - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): any additional index creation options (see the above list) should be passed as keyword arguments .. versionchanged:: 3.6 Added ``session`` parameter. Added support for passing maxTimeMS in kwargs. .. versionchanged:: 3.4 Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. Support the `collation` option. .. versionchanged:: 3.2 Added partialFilterExpression to support partial indexes. .. versionchanged:: 3.0 Renamed `key_or_list` to `keys`. Removed the `cache_for` option. :meth:`create_index` no longer caches index names. Removed support for the drop_dups and bucket_size aliases. .. mongodoc:: indexes """ keys = helpers._index_list(keys) name = kwargs.setdefault("name", helpers._gen_index_name(keys)) cmd_options = {} if "maxTimeMS" in kwargs: cmd_options["maxTimeMS"] = kwargs.pop("maxTimeMS") self.__create_index(keys, kwargs, session, **cmd_options) return name def ensure_index(self, key_or_list, cache_for=300, **kwargs): """**DEPRECATED** - Ensures that an index exists on this collection. .. versionchanged:: 3.0 **DEPRECATED** """ warnings.warn("ensure_index is deprecated. Use create_index instead.", DeprecationWarning, stacklevel=2) # The types supported by datetime.timedelta. if not (isinstance(cache_for, integer_types) or isinstance(cache_for, float)): raise TypeError("cache_for must be an integer or float.") if "drop_dups" in kwargs: kwargs["dropDups"] = kwargs.pop("drop_dups") if "bucket_size" in kwargs: kwargs["bucketSize"] = kwargs.pop("bucket_size") keys = helpers._index_list(key_or_list) name = kwargs.setdefault("name", helpers._gen_index_name(keys)) # Note that there is a race condition here. One thread could # check if the index is cached and be preempted before creating # and caching the index. This means multiple threads attempting # to create the same index concurrently could send the index # to the server two or more times. This has no practical impact # other than wasted round trips. if not self.__database.client._cached(self.__database.name, self.__name, name): self.__create_index(keys, kwargs, session=None) self.__database.client._cache_index(self.__database.name, self.__name, name, cache_for) return name return None def drop_indexes(self, session=None, **kwargs): """Drops all indexes on this collection. Can be used on non-existant collections or collections with no indexes. Raises OperationFailure on an error. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): optional arguments to the createIndexes command (like maxTimeMS) can be passed as keyword arguments. .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. .. versionchanged:: 3.6 Added ``session`` parameter. Added support for arbitrary keyword arguments. .. versionchanged:: 3.4 Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. """ self.__database.client._purge_index(self.__database.name, self.__name) self.drop_index("*", session=session, **kwargs) def drop_index(self, index_or_name, session=None, **kwargs): """Drops the specified index on this collection. Can be used on non-existant collections or collections with no indexes. Raises OperationFailure on an error (e.g. trying to drop an index that does not exist). `index_or_name` can be either an index name (as returned by `create_index`), or an index specifier (as passed to `create_index`). An index specifier should be a list of (key, direction) pairs. Raises TypeError if index is not an instance of (str, unicode, list). .. warning:: if a custom name was used on index creation (by passing the `name` parameter to :meth:`create_index` or :meth:`ensure_index`) the index **must** be dropped by name. :Parameters: - `index_or_name`: index (or name of index) to drop - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): optional arguments to the createIndexes command (like maxTimeMS) can be passed as keyword arguments. .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. .. versionchanged:: 3.6 Added ``session`` parameter. Added support for arbitrary keyword arguments. .. versionchanged:: 3.4 Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. """ name = index_or_name if isinstance(index_or_name, list): name = helpers._gen_index_name(index_or_name) if not isinstance(name, string_type): raise TypeError("index_or_name must be an index name or list") self.__database.client._purge_index( self.__database.name, self.__name, name) cmd = SON([("dropIndexes", self.__name), ("index", name)]) cmd.update(kwargs) with self._socket_for_writes() as sock_info: self._command(sock_info, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found"], write_concern=self.write_concern, parse_write_concern_error=True, session=session) def reindex(self, session=None, **kwargs): """Rebuilds all indexes on this collection. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): optional arguments to the reIndex command (like maxTimeMS) can be passed as keyword arguments. .. warning:: reindex blocks all other operations (indexes are built in the foreground) and will be slow for large collections. .. versionchanged:: 3.6 Added ``session`` parameter. Added support for arbitrary keyword arguments. .. versionchanged:: 3.4 Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. .. versionchanged:: 3.5 We no longer apply this collection's write concern to this operation. MongoDB 3.4 silently ignored the write concern. MongoDB 3.6+ returns an error if we include the write concern. """ cmd = SON([("reIndex", self.__name)]) cmd.update(kwargs) with self._socket_for_writes() as sock_info: return self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY, parse_write_concern_error=True, session=session) def list_indexes(self, session=None): """Get a cursor over the index documents for this collection. >>> for index in db.test.list_indexes(): ... print(index) ... SON([(u'v', 1), (u'key', SON([(u'_id', 1)])), (u'name', u'_id_'), (u'ns', u'test.test')]) :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: An instance of :class:`~pymongo.command_cursor.CommandCursor`. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionadded:: 3.0 """ codec_options = CodecOptions(SON) coll = self.with_options(codec_options) with self._socket_for_primary_reads() as (sock_info, slave_ok): cmd = SON([("listIndexes", self.__name), ("cursor", {})]) if sock_info.max_wire_version > 2: with self.__database.client._tmp_session(session, False) as s: try: cursor = self._command(sock_info, cmd, slave_ok, ReadPreference.PRIMARY, codec_options, session=s)["cursor"] except OperationFailure as exc: # Ignore NamespaceNotFound errors to match the behavior # of reading from *.system.indexes. if exc.code != 26: raise cursor = {'id': 0, 'firstBatch': []} return CommandCursor(coll, cursor, sock_info.address, session=s, explicit_session=session is not None) else: res = message._first_batch( sock_info, self.__database.name, "system.indexes", {"ns": self.__full_name}, 0, slave_ok, codec_options, ReadPreference.PRIMARY, cmd, self.database.client._event_listeners) cursor = res["cursor"] # Note that a collection can only have 64 indexes, so there # will never be a getMore call. return CommandCursor(coll, cursor, sock_info.address) def index_information(self, session=None): """Get information on this collection's indexes. Returns a dictionary where the keys are index names (as returned by create_index()) and the values are dictionaries containing information about each index. The dictionary is guaranteed to contain at least a single key, ``"key"`` which is a list of (key, direction) pairs specifying the index (as passed to create_index()). It will also contain any other metadata about the indexes, except for the ``"ns"`` and ``"name"`` keys, which are cleaned. Example output might look like this: >>> db.test.create_index("x", unique=True) u'x_1' >>> db.test.index_information() {u'_id_': {u'key': [(u'_id', 1)]}, u'x_1': {u'unique': True, u'key': [(u'x', 1)]}} :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. """ cursor = self.list_indexes(session=session) info = {} for index in cursor: index["key"] = index["key"].items() index = dict(index) info[index.pop("name")] = index return info def options(self, session=None): """Get the options set on this collection. Returns a dictionary of options and their values - see :meth:`~pymongo.database.Database.create_collection` for more information on the possible options. Returns an empty dictionary if the collection has not been created yet. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. .. versionchanged:: 3.6 Added ``session`` parameter. """ cursor = self.__database.list_collections( session=session, filter={"name": self.__name}) result = None for doc in cursor: result = doc break if not result: return {} options = result.get("options", {}) if "create" in options: del options["create"] return options def _aggregate(self, pipeline, cursor_class, first_batch_size, session, explicit_session, **kwargs): common.validate_list('pipeline', pipeline) if "explain" in kwargs: raise ConfigurationError("The explain option is not supported. " "Use Database.command instead.") collation = validate_collation_or_none(kwargs.pop('collation', None)) max_await_time_ms = kwargs.pop('maxAwaitTimeMS', None) cmd = SON([("aggregate", self.__name), ("pipeline", pipeline)]) # Remove things that are not command options. use_cursor = True if "useCursor" in kwargs: warnings.warn( "The useCursor option is deprecated " "and will be removed in PyMongo 4.0", DeprecationWarning, stacklevel=2) use_cursor = common.validate_boolean( "useCursor", kwargs.pop("useCursor")) batch_size = common.validate_non_negative_integer_or_none( "batchSize", kwargs.pop("batchSize", None)) # If the server does not support the "cursor" option we # ignore useCursor and batchSize. with self._socket_for_reads() as (sock_info, slave_ok): dollar_out = pipeline and '$out' in pipeline[-1] if use_cursor: if "cursor" not in kwargs: kwargs["cursor"] = {} # Ignore batchSize when the $out pipeline stage is used. # batchSize is meaningless in that case since the server # doesn't return results. This also avoids SERVER-23923. if first_batch_size is not None and not dollar_out: kwargs["cursor"]["batchSize"] = first_batch_size if (sock_info.max_wire_version >= 5 and dollar_out and self.write_concern): cmd['writeConcern'] = self.write_concern.document cmd.update(kwargs) # Apply this Collection's read concern if $out is not in the # pipeline. if (sock_info.max_wire_version >= 4 and 'readConcern' not in cmd and not dollar_out): read_concern = self.read_concern else: read_concern = None # Avoid auto-injecting a session: aggregate() passes a session, # aggregate_raw_batches() passes none. result = sock_info.command( self.__database.name, cmd, slave_ok, self.read_preference, self.codec_options, parse_write_concern_error=dollar_out, read_concern=read_concern, collation=collation, session=session, client=self.__database.client) if "cursor" in result: cursor = result["cursor"] else: # Pre-MongoDB 2.6. Fake a cursor. cursor = { "id": 0, "firstBatch": result["result"], "ns": self.full_name, } return cursor_class( self, cursor, sock_info.address, batch_size=batch_size or 0, max_await_time_ms=max_await_time_ms, session=session, explicit_session=explicit_session) def aggregate(self, pipeline, session=None, **kwargs): """Perform an aggregation using the aggregation framework on this collection. All optional `aggregate command`_ parameters should be passed as keyword arguments to this method. Valid options include, but are not limited to: - `allowDiskUse` (bool): Enables writing to temporary files. When set to True, aggregation stages can write data to the _tmp subdirectory of the --dbpath directory. The default is False. - `maxTimeMS` (int): The maximum amount of time to allow the operation to run in milliseconds. - `batchSize` (int): The maximum number of documents to return per batch. Ignored if the connected mongod or mongos does not support returning aggregate results using a cursor, or `useCursor` is ``False``. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `useCursor` (bool): Deprecated. Will be removed in PyMongo 4.0. The :meth:`aggregate` method obeys the :attr:`read_preference` of this :class:`Collection`. Please note that using the ``$out`` pipeline stage requires a read preference of :attr:`~pymongo.read_preferences.ReadPreference.PRIMARY` (the default). The server will raise an error if the ``$out`` pipeline stage is used with any other read preference. .. note:: This method does not support the 'explain' option. Please use :meth:`~pymongo.database.Database.command` instead. An example is included in the :ref:`aggregate-examples` documentation. .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. :Parameters: - `pipeline`: a list of aggregation pipeline stages - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): See list of options above. :Returns: A :class:`~pymongo.command_cursor.CommandCursor` over the result set. .. versionchanged:: 3.6 Added the `session` parameter. Added the `maxAwaitTimeMS` option. Deprecated the `useCursor` option. .. versionchanged:: 3.4 Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. Support the `collation` option. .. versionchanged:: 3.0 The :meth:`aggregate` method always returns a CommandCursor. The pipeline argument must be a list. .. versionchanged:: 2.7 When the cursor option is used, return :class:`~pymongo.command_cursor.CommandCursor` instead of :class:`~pymongo.cursor.Cursor`. .. versionchanged:: 2.6 Added cursor support. .. versionadded:: 2.3 .. seealso:: :doc:`/examples/aggregation` .. _aggregate command: https://docs.mongodb.com/manual/reference/command/aggregate """ with self.__database.client._tmp_session(session, close=False) as s: return self._aggregate(pipeline, CommandCursor, kwargs.get('batchSize'), session=s, explicit_session=session is not None, **kwargs) def aggregate_raw_batches(self, pipeline, **kwargs): """Perform an aggregation and retrieve batches of raw BSON. Similar to the :meth:`aggregate` method but returns a :class:`~pymongo.cursor.RawBatchCursor`. This example demonstrates how to work with raw batches, but in practice raw batches should be passed to an external library that can decode BSON into another data type, rather than used with PyMongo's :mod:`bson` module. >>> import bson >>> cursor = db.test.aggregate_raw_batches([ ... {'$project': {'x': {'$multiply': [2, '$x']}}}]) >>> for batch in cursor: ... print(bson.decode_all(batch)) .. note:: aggregate_raw_batches does not support sessions. .. versionadded:: 3.6 """ # OP_MSG with document stream returns is required to support # sessions. if "session" in kwargs: raise ConfigurationError( "aggregate_raw_batches does not support sessions") return self._aggregate(pipeline, RawBatchCommandCursor, 0, None, False, **kwargs) def watch(self, pipeline=None, full_document='default', resume_after=None, max_await_time_ms=None, batch_size=None, collation=None, session=None): """Watch changes on this collection. Performs an aggregation with an implicit initial ``$changeStream`` stage and returns a :class:`~pymongo.change_stream.ChangeStream` cursor which iterates over changes on this collection. Introduced in MongoDB 3.6. .. code-block:: python with db.collection.watch() as stream: for change in stream: print(change) The :class:`~pymongo.change_stream.ChangeStream` iterable blocks until the next change document is returned or an error is raised. If the :meth:`~pymongo.change_stream.ChangeStream.next` method encounters a network error when retrieving a batch from the server, it will automatically attempt to recreate the cursor such that no change events are missed. Any error encountered during the resume attempt indicates there may be an outage and will be raised. .. code-block:: python try: with db.collection.watch( [{'$match': {'operationType': 'insert'}}]) as stream: for insert_change in stream: print(insert_change) except pymongo.errors.PyMongoError: # The ChangeStream encountered an unrecoverable error or the # resume attempt failed to recreate the cursor. logging.error('...') For a precise description of the resume process see the `change streams specification`_. .. note:: Using this helper method is preferred to directly calling :meth:`~pymongo.collection.Collection.aggregate` with a ``$changeStream`` stage, for the purpose of supporting resumability. .. warning:: This Collection's :attr:`read_concern` must be ``ReadConcern("majority")`` in order to use the ``$changeStream`` stage. :Parameters: - `pipeline` (optional): A list of aggregation pipeline stages to append to an initial ``$changeStream`` stage. Not all pipeline stages are valid after a ``$changeStream`` stage, see the MongoDB documentation on change streams for the supported stages. - `full_document` (optional): The fullDocument to pass as an option to the ``$changeStream`` stage. Allowed values: 'default', 'updateLookup'. Defaults to 'default'. When set to 'updateLookup', the change notification for partial updates will include both a delta describing the changes to the document, as well as a copy of the entire document that was changed from some time after the change occurred. - `resume_after` (optional): The logical starting point for this change stream. - `max_await_time_ms` (optional): The maximum time in milliseconds for the server to wait for changes before responding to a getMore operation. - `batch_size` (optional): The maximum number of documents to return per batch. - `collation` (optional): The :class:`~pymongo.collation.Collation` to use for the aggregation. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: A :class:`~pymongo.change_stream.ChangeStream` cursor. .. versionadded:: 3.6 .. mongodoc:: changeStreams .. _change streams specification: https://github.com/mongodb/specifications/blob/master/source/change-streams.rst """ if pipeline is None: pipeline = [] else: if not isinstance(pipeline, list): raise TypeError("pipeline must be a list") common.validate_string_or_none('full_document', full_document) return ChangeStream(self, pipeline, full_document, resume_after, max_await_time_ms, batch_size, collation, session) def group(self, key, condition, initial, reduce, finalize=None, **kwargs): """Perform a query similar to an SQL *group by* operation. **DEPRECATED** - The group command was deprecated in MongoDB 3.4. The :meth:`~group` method is deprecated and will be removed in PyMongo 4.0. Use :meth:`~aggregate` with the `$group` stage or :meth:`~map_reduce` instead. .. versionchanged:: 3.5 Deprecated the group method. .. versionchanged:: 3.4 Added the `collation` option. .. versionchanged:: 2.2 Removed deprecated argument: command """ warnings.warn("The group method is deprecated and will be removed in " "PyMongo 4.0. Use the aggregate method with the $group " "stage or the map_reduce method instead.", DeprecationWarning, stacklevel=2) group = {} if isinstance(key, string_type): group["$keyf"] = Code(key) elif key is not None: group = {"key": helpers._fields_list_to_dict(key, "key")} group["ns"] = self.__name group["$reduce"] = Code(reduce) group["cond"] = condition group["initial"] = initial if finalize is not None: group["finalize"] = Code(finalize) cmd = SON([("group", group)]) collation = validate_collation_or_none(kwargs.pop('collation', None)) cmd.update(kwargs) with self._socket_for_reads() as (sock_info, slave_ok): return self._command(sock_info, cmd, slave_ok, collation=collation)["retval"] def rename(self, new_name, session=None, **kwargs): """Rename this collection. If operating in auth mode, client must be authorized as an admin to perform this operation. Raises :class:`TypeError` if `new_name` is not an instance of :class:`basestring` (:class:`str` in python 3). Raises :class:`~pymongo.errors.InvalidName` if `new_name` is not a valid collection name. :Parameters: - `new_name`: new name for this collection - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional arguments to the rename command may be passed as keyword arguments to this helper method (i.e. ``dropTarget=True``) .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. """ if not isinstance(new_name, string_type): raise TypeError("new_name must be an " "instance of %s" % (string_type.__name__,)) if not new_name or ".." in new_name: raise InvalidName("collection names cannot be empty") if new_name[0] == "." or new_name[-1] == ".": raise InvalidName("collecion names must not start or end with '.'") if "$" in new_name and not new_name.startswith("oplog.$main"): raise InvalidName("collection names must not contain '$'") new_name = "%s.%s" % (self.__database.name, new_name) cmd = SON([("renameCollection", self.__full_name), ("to", new_name)]) with self._socket_for_writes() as sock_info: with self.__database.client._tmp_session(session) as s: if sock_info.max_wire_version >= 5 and self.write_concern: cmd['writeConcern'] = self.write_concern.document cmd.update(kwargs) return sock_info.command( 'admin', cmd, parse_write_concern_error=True, session=s, client=self.__database.client) def distinct(self, key, filter=None, session=None, **kwargs): """Get a list of distinct values for `key` among all documents in this collection. Raises :class:`TypeError` if `key` is not an instance of :class:`basestring` (:class:`str` in python 3). All optional distinct parameters should be passed as keyword arguments to this method. Valid options include: - `maxTimeMS` (int): The maximum amount of time to allow the count command to run, in milliseconds. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. The :meth:`distinct` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `key`: name of the field for which we want to get the distinct values - `filter` (optional): A query document that specifies the documents from which to retrieve the distinct values. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): See list of options above. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Support the `collation` option. """ if not isinstance(key, string_type): raise TypeError("key must be an " "instance of %s" % (string_type.__name__,)) cmd = SON([("distinct", self.__name), ("key", key)]) if filter is not None: if "query" in kwargs: raise ConfigurationError("can't pass both filter and query") kwargs["query"] = filter collation = validate_collation_or_none(kwargs.pop('collation', None)) cmd.update(kwargs) with self._socket_for_reads() as (sock_info, slave_ok): return self._command(sock_info, cmd, slave_ok, read_concern=self.read_concern, collation=collation, session=session)["values"] def map_reduce(self, map, reduce, out, full_response=False, session=None, **kwargs): """Perform a map/reduce operation on this collection. If `full_response` is ``False`` (default) returns a :class:`~pymongo.collection.Collection` instance containing the results of the operation. Otherwise, returns the full response from the server to the `map reduce command`_. :Parameters: - `map`: map function (as a JavaScript string) - `reduce`: reduce function (as a JavaScript string) - `out`: output collection name or `out object` (dict). See the `map reduce command`_ documentation for available options. Note: `out` options are order sensitive. :class:`~bson.son.SON` can be used to specify multiple options. e.g. SON([('replace', ), ('db', )]) - `full_response` (optional): if ``True``, return full response to this command - otherwise just return the result collection - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional arguments to the `map reduce command`_ may be passed as keyword arguments to this helper method, e.g.:: >>> db.test.map_reduce(map, reduce, "myresults", limit=2) .. note:: The :meth:`map_reduce` method does **not** obey the :attr:`read_preference` of this :class:`Collection`. To run mapReduce on a secondary use the :meth:`inline_map_reduce` method instead. .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation (if the output is not inline) when using MongoDB >= 3.4. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Apply this collection's write concern automatically to this operation when connected to MongoDB >= 3.4. .. seealso:: :doc:`/examples/aggregation` .. versionchanged:: 3.4 Added the `collation` option. .. versionchanged:: 2.2 Removed deprecated arguments: merge_output and reduce_output .. _map reduce command: http://docs.mongodb.org/manual/reference/command/mapReduce/ .. mongodoc:: mapreduce """ if not isinstance(out, (string_type, abc.Mapping)): raise TypeError("'out' must be an instance of " "%s or a mapping" % (string_type.__name__,)) cmd = SON([("mapreduce", self.__name), ("map", map), ("reduce", reduce), ("out", out)]) collation = validate_collation_or_none(kwargs.pop('collation', None)) cmd.update(kwargs) inline = 'inline' in cmd['out'] with self._socket_for_primary_reads() as (sock_info, slave_ok): if (sock_info.max_wire_version >= 5 and self.write_concern and not inline): cmd['writeConcern'] = self.write_concern.document cmd.update(kwargs) if (sock_info.max_wire_version >= 4 and 'readConcern' not in cmd and inline): # No need to parse 'writeConcernError' here, since the command # is an inline map reduce. response = self._command( sock_info, cmd, slave_ok, ReadPreference.PRIMARY, read_concern=self.read_concern, collation=collation, session=session) else: response = self._command( sock_info, cmd, slave_ok, ReadPreference.PRIMARY, parse_write_concern_error=not inline, collation=collation, session=session) if full_response or not response.get('result'): return response elif isinstance(response['result'], dict): dbase = response['result']['db'] coll = response['result']['collection'] return self.__database.client[dbase][coll] else: return self.__database[response["result"]] def inline_map_reduce(self, map, reduce, full_response=False, session=None, **kwargs): """Perform an inline map/reduce operation on this collection. Perform the map/reduce operation on the server in RAM. A result collection is not created. The result set is returned as a list of documents. If `full_response` is ``False`` (default) returns the result documents in a list. Otherwise, returns the full response from the server to the `map reduce command`_. The :meth:`inline_map_reduce` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `map`: map function (as a JavaScript string) - `reduce`: reduce function (as a JavaScript string) - `full_response` (optional): if ``True``, return full response to this command - otherwise just return the result collection - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional arguments to the `map reduce command`_ may be passed as keyword arguments to this helper method, e.g.:: >>> db.test.inline_map_reduce(map, reduce, limit=2) .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Added the `collation` option. """ cmd = SON([("mapreduce", self.__name), ("map", map), ("reduce", reduce), ("out", {"inline": 1})]) collation = validate_collation_or_none(kwargs.pop('collation', None)) cmd.update(kwargs) with self._socket_for_reads() as (sock_info, slave_ok): if sock_info.max_wire_version >= 4 and 'readConcern' not in cmd: res = self._command(sock_info, cmd, slave_ok, read_concern=self.read_concern, collation=collation, session=session) else: res = self._command(sock_info, cmd, slave_ok, collation=collation, session=session) if full_response: return res else: return res.get("results") def __find_and_modify(self, filter, projection, sort, upsert=None, return_document=ReturnDocument.BEFORE, array_filters=None, session=None, **kwargs): """Internal findAndModify helper.""" common.validate_is_mapping("filter", filter) if not isinstance(return_document, bool): raise ValueError("return_document must be " "ReturnDocument.BEFORE or ReturnDocument.AFTER") collation = validate_collation_or_none(kwargs.pop('collation', None)) cmd = SON([("findAndModify", self.__name), ("query", filter), ("new", return_document)]) cmd.update(kwargs) if projection is not None: cmd["fields"] = helpers._fields_list_to_dict(projection, "projection") if sort is not None: cmd["sort"] = helpers._index_document(sort) if upsert is not None: common.validate_boolean("upsert", upsert) cmd["upsert"] = upsert write_concern = cmd.get('writeConcern') if write_concern is not None: acknowledged = write_concern.get("w") != 0 else: acknowledged = self.write_concern.acknowledged def _find_and_modify(session, sock_info, retryable_write): if array_filters is not None: if sock_info.max_wire_version < 6: raise ConfigurationError( 'Must be connected to MongoDB 3.6+ to use ' 'arrayFilters.') if not self.write_concern.acknowledged: raise ConfigurationError( 'arrayFilters is unsupported for unacknowledged ' 'writes.') cmd["arrayFilters"] = array_filters if sock_info.max_wire_version >= 4 and 'writeConcern' not in cmd: wc_doc = self.write_concern.document if wc_doc: cmd['writeConcern'] = wc_doc out = self._command(sock_info, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=[_NO_OBJ_ERROR], collation=collation, session=session, retryable_write=retryable_write) _check_write_command_response(out) return out.get("value") return self.__database.client._retryable_write( acknowledged, _find_and_modify, session) def find_one_and_delete(self, filter, projection=None, sort=None, session=None, **kwargs): """Finds a single document and deletes it, returning the document. >>> db.test.count({'x': 1}) 2 >>> db.test.find_one_and_delete({'x': 1}) {u'x': 1, u'_id': ObjectId('54f4e12bfba5220aa4d6dee8')} >>> db.test.count({'x': 1}) 1 If multiple documents match *filter*, a *sort* can be applied. >>> for doc in db.test.find({'x': 1}): ... print(doc) ... {u'x': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} >>> db.test.find_one_and_delete( ... {'x': 1}, sort=[('_id', pymongo.DESCENDING)]) {u'x': 1, u'_id': 2} The *projection* option can be used to limit the fields returned. >>> db.test.find_one_and_delete({'x': 1}, projection={'_id': False}) {u'x': 1} :Parameters: - `filter`: A query that matches the document to delete. - `projection` (optional): a list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a mapping to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is deleted. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.2 Respects write concern. .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionchanged:: 3.4 Added the `collation` option. .. versionadded:: 3.0 """ kwargs['remove'] = True return self.__find_and_modify(filter, projection, sort, session=session, **kwargs) def find_one_and_replace(self, filter, replacement, projection=None, sort=None, upsert=False, return_document=ReturnDocument.BEFORE, session=None, **kwargs): """Finds a single document and replaces it, returning either the original or the replaced document. The :meth:`find_one_and_replace` method differs from :meth:`find_one_and_update` by replacing the document matched by *filter*, rather than modifying the existing document. >>> for doc in db.test.find({}): ... print(doc) ... {u'x': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} >>> db.test.find_one_and_replace({'x': 1}, {'y': 1}) {u'x': 1, u'_id': 0} >>> for doc in db.test.find({}): ... print(doc) ... {u'y': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} :Parameters: - `filter`: A query that matches the document to replace. - `replacement`: The replacement document. - `projection` (optional): A list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a mapping to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is replaced. - `upsert` (optional): When ``True``, inserts a new document if no document matches the query. Defaults to ``False``. - `return_document`: If :attr:`ReturnDocument.BEFORE` (the default), returns the original document before it was replaced, or ``None`` if no document matches. If :attr:`ReturnDocument.AFTER`, returns the replaced or inserted document. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Added the `collation` option. .. versionchanged:: 3.2 Respects write concern. .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionadded:: 3.0 """ common.validate_ok_for_replace(replacement) kwargs['update'] = replacement return self.__find_and_modify(filter, projection, sort, upsert, return_document, session=session, **kwargs) def find_one_and_update(self, filter, update, projection=None, sort=None, upsert=False, return_document=ReturnDocument.BEFORE, array_filters=None, session=None, **kwargs): """Finds a single document and updates it, returning either the original or the updated document. >>> db.test.find_one_and_update( ... {'_id': 665}, {'$inc': {'count': 1}, '$set': {'done': True}}) {u'_id': 665, u'done': False, u'count': 25}} By default :meth:`find_one_and_update` returns the original version of the document before the update was applied. To return the updated version of the document instead, use the *return_document* option. >>> from pymongo import ReturnDocument >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... return_document=ReturnDocument.AFTER) {u'_id': u'userid', u'seq': 1} You can limit the fields returned with the *projection* option. >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... projection={'seq': True, '_id': False}, ... return_document=ReturnDocument.AFTER) {u'seq': 2} The *upsert* option can be used to create the document if it doesn't already exist. >>> db.example.delete_many({}).deleted_count 1 >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... projection={'seq': True, '_id': False}, ... upsert=True, ... return_document=ReturnDocument.AFTER) {u'seq': 1} If multiple documents match *filter*, a *sort* can be applied. >>> for doc in db.test.find({'done': True}): ... print(doc) ... {u'_id': 665, u'done': True, u'result': {u'count': 26}} {u'_id': 701, u'done': True, u'result': {u'count': 17}} >>> db.test.find_one_and_update( ... {'done': True}, ... {'$set': {'final': True}}, ... sort=[('_id', pymongo.DESCENDING)]) {u'_id': 701, u'done': True, u'result': {u'count': 17}} :Parameters: - `filter`: A query that matches the document to update. - `update`: The update operations to apply. - `projection` (optional): A list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a dict to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is updated. - `upsert` (optional): When ``True``, inserts a new document if no document matches the query. Defaults to ``False``. - `return_document`: If :attr:`ReturnDocument.BEFORE` (the default), returns the original document before it was updated, or ``None`` if no document matches. If :attr:`ReturnDocument.AFTER`, returns the updated or inserted document. - `array_filters` (optional): A list of filters specifying which array elements an update should apply. Requires MongoDB 3.6+. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). .. versionchanged:: 3.6 Added the `array_filters` and `session` options. .. versionchanged:: 3.4 Added the `collation` option. .. versionchanged:: 3.2 Respects write concern. .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionadded:: 3.0 """ common.validate_ok_for_update(update) common.validate_list_or_none('array_filters', array_filters) kwargs['update'] = update return self.__find_and_modify(filter, projection, sort, upsert, return_document, array_filters, session=session, **kwargs) def save(self, to_save, manipulate=True, check_keys=True, **kwargs): """Save a document in this collection. **DEPRECATED** - Use :meth:`insert_one` or :meth:`replace_one` instead. .. versionchanged:: 3.0 Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write operations. """ warnings.warn("save is deprecated. Use insert_one or replace_one " "instead", DeprecationWarning, stacklevel=2) common.validate_is_document_type("to_save", to_save) write_concern = None collation = validate_collation_or_none(kwargs.pop('collation', None)) if kwargs: write_concern = WriteConcern(**kwargs) if not (isinstance(to_save, RawBSONDocument) or "_id" in to_save): return self._insert( to_save, True, check_keys, manipulate, write_concern) else: self._update_retryable( {"_id": to_save["_id"]}, to_save, True, check_keys, False, manipulate, write_concern, collation=collation) return to_save.get("_id") def insert(self, doc_or_docs, manipulate=True, check_keys=True, continue_on_error=False, **kwargs): """Insert a document(s) into this collection. **DEPRECATED** - Use :meth:`insert_one` or :meth:`insert_many` instead. .. versionchanged:: 3.0 Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write operations. """ warnings.warn("insert is deprecated. Use insert_one or insert_many " "instead.", DeprecationWarning, stacklevel=2) write_concern = None if kwargs: write_concern = WriteConcern(**kwargs) return self._insert(doc_or_docs, not continue_on_error, check_keys, manipulate, write_concern) def update(self, spec, document, upsert=False, manipulate=False, multi=False, check_keys=True, **kwargs): """Update a document(s) in this collection. **DEPRECATED** - Use :meth:`replace_one`, :meth:`update_one`, or :meth:`update_many` instead. .. versionchanged:: 3.0 Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write operations. """ warnings.warn("update is deprecated. Use replace_one, update_one or " "update_many instead.", DeprecationWarning, stacklevel=2) common.validate_is_mapping("spec", spec) common.validate_is_mapping("document", document) if document: # If a top level key begins with '$' this is a modify operation # and we should skip key validation. It doesn't matter which key # we check here. Passing a document with a mix of top level keys # starting with and without a '$' is invalid and the server will # raise an appropriate exception. first = next(iter(document)) if first.startswith('$'): check_keys = False write_concern = None collation = validate_collation_or_none(kwargs.pop('collation', None)) if kwargs: write_concern = WriteConcern(**kwargs) return self._update_retryable( spec, document, upsert, check_keys, multi, manipulate, write_concern, collation=collation) def remove(self, spec_or_id=None, multi=True, **kwargs): """Remove a document(s) from this collection. **DEPRECATED** - Use :meth:`delete_one` or :meth:`delete_many` instead. .. versionchanged:: 3.0 Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write operations. """ warnings.warn("remove is deprecated. Use delete_one or delete_many " "instead.", DeprecationWarning, stacklevel=2) if spec_or_id is None: spec_or_id = {} if not isinstance(spec_or_id, abc.Mapping): spec_or_id = {"_id": spec_or_id} write_concern = None collation = validate_collation_or_none(kwargs.pop('collation', None)) if kwargs: write_concern = WriteConcern(**kwargs) return self._delete_retryable( spec_or_id, multi, write_concern, collation=collation) def find_and_modify(self, query={}, update=None, upsert=False, sort=None, full_response=False, manipulate=False, **kwargs): """Update and return an object. **DEPRECATED** - Use :meth:`find_one_and_delete`, :meth:`find_one_and_replace`, or :meth:`find_one_and_update` instead. """ warnings.warn("find_and_modify is deprecated, use find_one_and_delete" ", find_one_and_replace, or find_one_and_update instead", DeprecationWarning, stacklevel=2) if not update and not kwargs.get('remove', None): raise ValueError("Must either update or remove") if update and kwargs.get('remove', None): raise ValueError("Can't do both update and remove") # No need to include empty args if query: kwargs['query'] = query if update: kwargs['update'] = update if upsert: kwargs['upsert'] = upsert if sort: # Accept a list of tuples to match Cursor's sort parameter. if isinstance(sort, list): kwargs['sort'] = helpers._index_document(sort) # Accept OrderedDict, SON, and dict with len == 1 so we # don't break existing code already using find_and_modify. elif (isinstance(sort, ORDERED_TYPES) or isinstance(sort, dict) and len(sort) == 1): warnings.warn("Passing mapping types for `sort` is deprecated," " use a list of (key, direction) pairs instead", DeprecationWarning, stacklevel=2) kwargs['sort'] = sort else: raise TypeError("sort must be a list of (key, direction) " "pairs, a dict of len 1, or an instance of " "SON or OrderedDict") fields = kwargs.pop("fields", None) if fields is not None: kwargs["fields"] = helpers._fields_list_to_dict(fields, "fields") collation = validate_collation_or_none(kwargs.pop('collation', None)) cmd = SON([("findAndModify", self.__name)]) cmd.update(kwargs) write_concern = cmd.get('writeConcern') if write_concern is not None: acknowledged = write_concern.get("w") != 0 else: acknowledged = self.write_concern.acknowledged def _find_and_modify(session, sock_info, retryable_write): if sock_info.max_wire_version >= 4 and 'writeConcern' not in cmd: wc_doc = self.write_concern.document if wc_doc: cmd['writeConcern'] = wc_doc return self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=[_NO_OBJ_ERROR], collation=collation, session=session, retryable_write=retryable_write) out = self.__database.client._retryable_write( acknowledged, _find_and_modify, None) _check_write_command_response(out) if not out['ok']: if out["errmsg"] == _NO_OBJ_ERROR: return None else: # Should never get here b/c of allowable_errors raise ValueError("Unexpected Error: %s" % (out,)) if full_response: return out else: document = out.get('value') if manipulate: document = self.__database._fix_outgoing(document, self) return document def __iter__(self): return self def __next__(self): raise TypeError("'Collection' object is not iterable") next = __next__ def __call__(self, *args, **kwargs): """This is only here so that some API misusages are easier to debug. """ if "." not in self.__name: raise TypeError("'Collection' object is not callable. If you " "meant to call the '%s' method on a 'Database' " "object it is failing because no such method " "exists." % self.__name) raise TypeError("'Collection' object is not callable. If you meant to " "call the '%s' method on a 'Collection' object it is " "failing because no such method exists." % self.__name.split(".")[-1]) pymongo-3.6.1/pymongo/change_stream.py0000644000076600000240000000676613245621354020321 0ustar shanestaff00000000000000# Copyright 2017 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """ChangeStream cursor to iterate over changes on a collection.""" import copy from pymongo.errors import (ConnectionFailure, CursorNotFound, InvalidOperation, PyMongoError) class ChangeStream(object): """A change stream cursor. Should not be called directly by application developers. Use :meth:`~pymongo.collection.Collection.watch` instead. .. versionadded: 3.6 .. mongodoc:: changeStreams """ def __init__(self, collection, pipeline, full_document, resume_after=None, max_await_time_ms=None, batch_size=None, collation=None, session=None): self._collection = collection self._pipeline = copy.deepcopy(pipeline) self._full_document = full_document self._resume_token = copy.deepcopy(resume_after) self._max_await_time_ms = max_await_time_ms self._batch_size = batch_size self._collation = collation self._session = session self._cursor = self._create_cursor() def _full_pipeline(self): """Return the full aggregation pipeline for this ChangeStream.""" options = {} if self._full_document is not None: options['fullDocument'] = self._full_document if self._resume_token is not None: options['resumeAfter'] = self._resume_token full_pipeline = [{'$changeStream': options}] full_pipeline.extend(self._pipeline) return full_pipeline def _create_cursor(self): """Initialize the cursor or raise a fatal error""" return self._collection.aggregate( self._full_pipeline(), self._session, batchSize=self._batch_size, collation=self._collation, maxAwaitTimeMS=self._max_await_time_ms) def close(self): """Close this ChangeStream.""" self._cursor.close() def __iter__(self): return self def next(self): """Advance the cursor. This method blocks until the next change document is returned or an unrecoverable error is raised. Raises :exc:`StopIteration` if this ChangeStream is closed. """ while True: try: change = self._cursor.next() except (ConnectionFailure, CursorNotFound): try: self._cursor.close() except PyMongoError: pass self._cursor = self._create_cursor() continue try: resume_token = change['_id'] except KeyError: self.close() raise InvalidOperation( "Cannot provide resume functionality when the resume " "token is missing.") self._resume_token = copy.copy(resume_token) return change __next__ = next def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() pymongo-3.6.1/pymongo/settings.py0000644000076600000240000000752513245621354017353 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Represent MongoClient's configuration.""" import threading from bson.objectid import ObjectId from pymongo import common, monitor, pool from pymongo.common import LOCAL_THRESHOLD_MS, SERVER_SELECTION_TIMEOUT from pymongo.errors import ConfigurationError from pymongo.topology_description import TOPOLOGY_TYPE from pymongo.pool import PoolOptions from pymongo.server_description import ServerDescription class TopologySettings(object): def __init__(self, seeds=None, replica_set_name=None, pool_class=None, pool_options=None, monitor_class=None, condition_class=None, local_threshold_ms=LOCAL_THRESHOLD_MS, server_selection_timeout=SERVER_SELECTION_TIMEOUT, heartbeat_frequency=common.HEARTBEAT_FREQUENCY): """Represent MongoClient's configuration. Take a list of (host, port) pairs and optional replica set name. """ if heartbeat_frequency < common.MIN_HEARTBEAT_INTERVAL: raise ConfigurationError( "heartbeatFrequencyMS cannot be less than %d" % ( common.MIN_HEARTBEAT_INTERVAL * 1000,)) self._seeds = seeds or [('localhost', 27017)] self._replica_set_name = replica_set_name self._pool_class = pool_class or pool.Pool self._pool_options = pool_options or PoolOptions() self._monitor_class = monitor_class or monitor.Monitor self._condition_class = condition_class or threading.Condition self._local_threshold_ms = local_threshold_ms self._server_selection_timeout = server_selection_timeout self._heartbeat_frequency = heartbeat_frequency self._direct = (len(self._seeds) == 1 and not replica_set_name) self._topology_id = ObjectId() @property def seeds(self): """List of server addresses.""" return self._seeds @property def replica_set_name(self): return self._replica_set_name @property def pool_class(self): return self._pool_class @property def pool_options(self): return self._pool_options @property def monitor_class(self): return self._monitor_class @property def condition_class(self): return self._condition_class @property def local_threshold_ms(self): return self._local_threshold_ms @property def server_selection_timeout(self): return self._server_selection_timeout @property def heartbeat_frequency(self): return self._heartbeat_frequency @property def direct(self): """Connect directly to a single server, or use a set of servers? True if there is one seed and no replica_set_name. """ return self._direct def get_topology_type(self): if self.direct: return TOPOLOGY_TYPE.Single elif self.replica_set_name is not None: return TOPOLOGY_TYPE.ReplicaSetNoPrimary else: return TOPOLOGY_TYPE.Unknown def get_server_descriptions(self): """Initial dict of (address, ServerDescription) for all seeds.""" return dict([ (address, ServerDescription(address)) for address in self.seeds]) pymongo-3.6.1/pymongo/ssl_context.py0000644000076600000240000000711613245621354020054 0ustar shanestaff00000000000000# Copyright 2014-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """A fake SSLContext implementation.""" try: import ssl except ImportError: pass class SSLContext(object): """A fake SSLContext. This implements an API similar to ssl.SSLContext from python 3.2 but does not implement methods or properties that would be incompatible with ssl.wrap_socket from python 2.6. You must pass protocol which must be one of the PROTOCOL_* constants defined in the ssl module. ssl.PROTOCOL_SSLv23 is recommended for maximum interoperability. """ __slots__ = ('_cafile', '_certfile', '_keyfile', '_protocol', '_verify_mode') def __init__(self, protocol): self._cafile = None self._certfile = None self._keyfile = None self._protocol = protocol self._verify_mode = ssl.CERT_NONE @property def protocol(self): """The protocol version chosen when constructing the context. This attribute is read-only. """ return self._protocol def __get_verify_mode(self): """Whether to try to verify other peers' certificates and how to behave if verification fails. This attribute must be one of ssl.CERT_NONE, ssl.CERT_OPTIONAL or ssl.CERT_REQUIRED. """ return self._verify_mode def __set_verify_mode(self, value): """Setter for verify_mode.""" self._verify_mode = value verify_mode = property(__get_verify_mode, __set_verify_mode) def load_cert_chain(self, certfile, keyfile=None): """Load a private key and the corresponding certificate. The certfile string must be the path to a single file in PEM format containing the certificate as well as any number of CA certificates needed to establish the certificate's authenticity. The keyfile string, if present, must point to a file containing the private key. Otherwise the private key will be taken from certfile as well. """ self._certfile = certfile self._keyfile = keyfile def load_verify_locations(self, cafile=None, dummy=None): """Load a set of "certification authority"(CA) certificates used to validate other peers' certificates when `~verify_mode` is other than ssl.CERT_NONE. """ self._cafile = cafile def wrap_socket(self, sock, server_side=False, do_handshake_on_connect=True, suppress_ragged_eofs=True, dummy=None): """Wrap an existing Python socket sock and return an ssl.SSLSocket object. """ return ssl.wrap_socket(sock, keyfile=self._keyfile, certfile=self._certfile, server_side=server_side, cert_reqs=self._verify_mode, ssl_version=self._protocol, ca_certs=self._cafile, do_handshake_on_connect=do_handshake_on_connect, suppress_ragged_eofs=suppress_ragged_eofs) pymongo-3.6.1/pymongo/errors.py0000644000076600000240000001442113245621354017020 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Exceptions raised by PyMongo.""" from bson.errors import * try: from ssl import CertificateError except ImportError: from pymongo.ssl_match_hostname import CertificateError class PyMongoError(Exception): """Base class for all PyMongo exceptions.""" class ProtocolError(PyMongoError): """Raised for failures related to the wire protocol.""" class ConnectionFailure(PyMongoError): """Raised when a connection to the database cannot be made or is lost.""" class AutoReconnect(ConnectionFailure): """Raised when a connection to the database is lost and an attempt to auto-reconnect will be made. In order to auto-reconnect you must handle this exception, recognizing that the operation which caused it has not necessarily succeeded. Future operations will attempt to open a new connection to the database (and will continue to raise this exception until the first successful connection is made). Subclass of :exc:`~pymongo.errors.ConnectionFailure`. """ def __init__(self, message='', errors=None): self.errors = self.details = errors or [] ConnectionFailure.__init__(self, message) class NetworkTimeout(AutoReconnect): """An operation on an open connection exceeded socketTimeoutMS. The remaining connections in the pool stay open. In the case of a write operation, you cannot know whether it succeeded or failed. Subclass of :exc:`~pymongo.errors.AutoReconnect`. """ class NotMasterError(AutoReconnect): """The server responded "not master" or "node is recovering". These errors result from a query, write, or command. The operation failed because the client thought it was using the primary but the primary has stepped down, or the client thought it was using a healthy secondary but the secondary is stale and trying to recover. The client launches a refresh operation on a background thread, to update its view of the server as soon as possible after throwing this exception. Subclass of :exc:`~pymongo.errors.AutoReconnect`. """ class ServerSelectionTimeoutError(AutoReconnect): """Thrown when no MongoDB server is available for an operation If there is no suitable server for an operation PyMongo tries for ``serverSelectionTimeoutMS`` (default 30 seconds) to find one, then throws this exception. For example, it is thrown after attempting an operation when PyMongo cannot connect to any server, or if you attempt an insert into a replica set that has no primary and does not elect one within the timeout window, or if you attempt to query with a Read Preference that the replica set cannot satisfy. """ class ConfigurationError(PyMongoError): """Raised when something is incorrectly configured. """ class OperationFailure(PyMongoError): """Raised when a database operation fails. .. versionadded:: 2.7 The :attr:`details` attribute. """ def __init__(self, error, code=None, details=None): self.__code = code self.__details = details PyMongoError.__init__(self, error) @property def code(self): """The error code returned by the server, if any. """ return self.__code @property def details(self): """The complete error document returned by the server. Depending on the error that occurred, the error document may include useful information beyond just the error message. When connected to a mongos the error document may contain one or more subdocuments if errors occurred on multiple shards. """ return self.__details class CursorNotFound(OperationFailure): """Raised while iterating query results if the cursor is invalidated on the server. .. versionadded:: 2.7 """ class ExecutionTimeout(OperationFailure): """Raised when a database operation times out, exceeding the $maxTimeMS set in the query or command option. .. note:: Requires server version **>= 2.6.0** .. versionadded:: 2.7 """ class WriteConcernError(OperationFailure): """Base exception type for errors raised due to write concern. .. versionadded:: 3.0 """ class WriteError(OperationFailure): """Base exception type for errors raised during write operations. .. versionadded:: 3.0 """ class WTimeoutError(WriteConcernError): """Raised when a database operation times out (i.e. wtimeout expires) before replication completes. With newer versions of MongoDB the `details` attribute may include write concern fields like 'n', 'updatedExisting', or 'writtenTo'. .. versionadded:: 2.7 """ class DuplicateKeyError(WriteError): """Raised when an insert or update fails due to a duplicate key error.""" class BulkWriteError(OperationFailure): """Exception class for bulk write errors. .. versionadded:: 2.7 """ def __init__(self, results): OperationFailure.__init__( self, "batch op errors occurred", 65, results) class InvalidOperation(PyMongoError): """Raised when a client attempts to perform an invalid operation.""" class InvalidName(PyMongoError): """Raised when an invalid name is used.""" class CollectionInvalid(PyMongoError): """Raised when collection validation fails.""" class InvalidURI(ConfigurationError): """Raised when trying to parse an invalid mongodb URI.""" class ExceededMaxWaiters(Exception): """Raised when a thread tries to get a connection from a pool and ``maxPoolSize * waitQueueMultiple`` threads are already waiting. .. versionadded:: 2.6 """ pass class DocumentTooLarge(InvalidDocument): """Raised when an encoded document is too large for the connected server. """ pass pymongo-3.6.1/pymongo/read_preferences.py0000644000076600000240000004015613245621354021004 0ustar shanestaff00000000000000# Copyright 2012-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License", # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for choosing which member of a replica set to read from.""" from bson.py3compat import abc, integer_types from pymongo import max_staleness_selectors from pymongo.errors import ConfigurationError from pymongo.server_selectors import (member_with_tags_server_selector, secondary_with_tags_server_selector) _PRIMARY = 0 _PRIMARY_PREFERRED = 1 _SECONDARY = 2 _SECONDARY_PREFERRED = 3 _NEAREST = 4 _MONGOS_MODES = ( 'primary', 'primaryPreferred', 'secondary', 'secondaryPreferred', 'nearest', ) def _validate_tag_sets(tag_sets): """Validate tag sets for a MongoReplicaSetClient. """ if tag_sets is None: return tag_sets if not isinstance(tag_sets, list): raise TypeError(( "Tag sets %r invalid, must be a list") % (tag_sets,)) if len(tag_sets) == 0: raise ValueError(( "Tag sets %r invalid, must be None or contain at least one set of" " tags") % (tag_sets,)) for tags in tag_sets: if not isinstance(tags, abc.Mapping): raise TypeError( "Tag set %r invalid, must be an instance of dict, " "bson.son.SON or other type that inherits from " "collection.Mapping" % (tags,)) return tag_sets def _invalid_max_staleness_msg(max_staleness): return ("maxStalenessSeconds must be a positive integer, not %s" % max_staleness) # Some duplication with common.py to avoid import cycle. def _validate_max_staleness(max_staleness): """Validate max_staleness.""" if max_staleness == -1: return -1 if not isinstance(max_staleness, integer_types): raise TypeError(_invalid_max_staleness_msg(max_staleness)) if max_staleness <= 0: raise ValueError(_invalid_max_staleness_msg(max_staleness)) return max_staleness class _ServerMode(object): """Base class for all read preferences. """ __slots__ = ("__mongos_mode", "__mode", "__tag_sets", "__max_staleness") def __init__(self, mode, tag_sets=None, max_staleness=-1): self.__mongos_mode = _MONGOS_MODES[mode] self.__mode = mode self.__tag_sets = _validate_tag_sets(tag_sets) self.__max_staleness = _validate_max_staleness(max_staleness) @property def name(self): """The name of this read preference. """ return self.__class__.__name__ @property def mongos_mode(self): """The mongos mode of this read preference. """ return self.__mongos_mode @property def document(self): """Read preference as a document. """ doc = {'mode': self.__mongos_mode} if self.__tag_sets not in (None, [{}]): doc['tags'] = self.__tag_sets if self.__max_staleness != -1: doc['maxStalenessSeconds'] = self.__max_staleness return doc @property def mode(self): """The mode of this read preference instance. """ return self.__mode @property def tag_sets(self): """Set ``tag_sets`` to a list of dictionaries like [{'dc': 'ny'}] to read only from members whose ``dc`` tag has the value ``"ny"``. To specify a priority-order for tag sets, provide a list of tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag set, ``{}``, means "read from any member that matches the mode, ignoring tags." MongoReplicaSetClient tries each set of tags in turn until it finds a set of tags with at least one matching member. .. seealso:: `Data-Center Awareness `_ """ return list(self.__tag_sets) if self.__tag_sets else [{}] @property def max_staleness(self): """The maximum estimated length of time (in seconds) a replica set secondary can fall behind the primary in replication before it will no longer be selected for operations, or -1 for no maximum.""" return self.__max_staleness @property def min_wire_version(self): """The wire protocol version the server must support. Some read preferences impose version requirements on all servers (e.g. maxStalenessSeconds requires MongoDB 3.4 / maxWireVersion 5). All servers' maxWireVersion must be at least this read preference's `min_wire_version`, or the driver raises :exc:`~pymongo.errors.ConfigurationError`. """ return 0 if self.__max_staleness == -1 else 5 def __repr__(self): return "%s(tag_sets=%r, max_staleness=%r)" % ( self.name, self.__tag_sets, self.__max_staleness) def __eq__(self, other): if isinstance(other, _ServerMode): return (self.mode == other.mode and self.tag_sets == other.tag_sets and self.max_staleness == other.max_staleness) return NotImplemented def __ne__(self, other): return not self == other def __getstate__(self): """Return value of object for pickling. Needed explicitly because __slots__() defined. """ return {'mode': self.__mode, 'tag_sets': self.__tag_sets, 'max_staleness': self.__max_staleness} def __setstate__(self, value): """Restore from pickling.""" self.__mode = value['mode'] self.__mongos_mode = _MONGOS_MODES[self.__mode] self.__tag_sets = _validate_tag_sets(value['tag_sets']) self.__max_staleness = _validate_max_staleness(value['max_staleness']) class Primary(_ServerMode): """Primary read preference. * When directly connected to one mongod queries are allowed if the server is standalone or a replica set primary. * When connected to a mongos queries are sent to the primary of a shard. * When connected to a replica set queries are sent to the primary of the replica set. """ __slots__ = () def __init__(self): super(Primary, self).__init__(_PRIMARY) def __call__(self, selection): """Apply this read preference to a Selection.""" return selection.primary_selection def __repr__(self): return "Primary()" def __eq__(self, other): if isinstance(other, _ServerMode): return other.mode == _PRIMARY return NotImplemented class PrimaryPreferred(_ServerMode): """PrimaryPreferred read preference. * When directly connected to one mongod queries are allowed to standalone servers, to a replica set primary, or to replica set secondaries. * When connected to a mongos queries are sent to the primary of a shard if available, otherwise a shard secondary. * When connected to a replica set queries are sent to the primary if available, otherwise a secondary. :Parameters: - `tag_sets`: The :attr:`~tag_sets` to use if the primary is not available. - `max_staleness`: (integer, in seconds) The maximum estimated length of time a replica set secondary can fall behind the primary in replication before it will no longer be selected for operations. Default -1, meaning no maximum. If it is set, it must be at least 90 seconds. """ __slots__ = () def __init__(self, tag_sets=None, max_staleness=-1): super(PrimaryPreferred, self).__init__(_PRIMARY_PREFERRED, tag_sets, max_staleness) def __call__(self, selection): """Apply this read preference to Selection.""" if selection.primary: return selection.primary_selection else: return secondary_with_tags_server_selector( self.tag_sets, max_staleness_selectors.select( self.max_staleness, selection)) class Secondary(_ServerMode): """Secondary read preference. * When directly connected to one mongod queries are allowed to standalone servers, to a replica set primary, or to replica set secondaries. * When connected to a mongos queries are distributed among shard secondaries. An error is raised if no secondaries are available. * When connected to a replica set queries are distributed among secondaries. An error is raised if no secondaries are available. :Parameters: - `tag_sets`: The :attr:`~tag_sets` for this read preference. - `max_staleness`: (integer, in seconds) The maximum estimated length of time a replica set secondary can fall behind the primary in replication before it will no longer be selected for operations. Default -1, meaning no maximum. If it is set, it must be at least 90 seconds. """ __slots__ = () def __init__(self, tag_sets=None, max_staleness=-1): super(Secondary, self).__init__(_SECONDARY, tag_sets, max_staleness) def __call__(self, selection): """Apply this read preference to Selection.""" return secondary_with_tags_server_selector( self.tag_sets, max_staleness_selectors.select( self.max_staleness, selection)) class SecondaryPreferred(_ServerMode): """SecondaryPreferred read preference. * When directly connected to one mongod queries are allowed to standalone servers, to a replica set primary, or to replica set secondaries. * When connected to a mongos queries are distributed among shard secondaries, or the shard primary if no secondary is available. * When connected to a replica set queries are distributed among secondaries, or the primary if no secondary is available. :Parameters: - `tag_sets`: The :attr:`~tag_sets` for this read preference. - `max_staleness`: (integer, in seconds) The maximum estimated length of time a replica set secondary can fall behind the primary in replication before it will no longer be selected for operations. Default -1, meaning no maximum. If it is set, it must be at least 90 seconds. """ __slots__ = () def __init__(self, tag_sets=None, max_staleness=-1): super(SecondaryPreferred, self).__init__(_SECONDARY_PREFERRED, tag_sets, max_staleness) def __call__(self, selection): """Apply this read preference to Selection.""" secondaries = secondary_with_tags_server_selector( self.tag_sets, max_staleness_selectors.select( self.max_staleness, selection)) if secondaries: return secondaries else: return selection.primary_selection class Nearest(_ServerMode): """Nearest read preference. * When directly connected to one mongod queries are allowed to standalone servers, to a replica set primary, or to replica set secondaries. * When connected to a mongos queries are distributed among all members of a shard. * When connected to a replica set queries are distributed among all members. :Parameters: - `tag_sets`: The :attr:`~tag_sets` for this read preference. - `max_staleness`: (integer, in seconds) The maximum estimated length of time a replica set secondary can fall behind the primary in replication before it will no longer be selected for operations. Default -1, meaning no maximum. If it is set, it must be at least 90 seconds. """ __slots__ = () def __init__(self, tag_sets=None, max_staleness=-1): super(Nearest, self).__init__(_NEAREST, tag_sets, max_staleness) def __call__(self, selection): """Apply this read preference to Selection.""" return member_with_tags_server_selector( self.tag_sets, max_staleness_selectors.select( self.max_staleness, selection)) _ALL_READ_PREFERENCES = (Primary, PrimaryPreferred, Secondary, SecondaryPreferred, Nearest) def make_read_preference(mode, tag_sets, max_staleness=-1): if mode == _PRIMARY: if tag_sets not in (None, [{}]): raise ConfigurationError("Read preference primary " "cannot be combined with tags") if max_staleness != -1: raise ConfigurationError("Read preference primary cannot be " "combined with maxStalenessSeconds") return Primary() return _ALL_READ_PREFERENCES[mode](tag_sets, max_staleness) _MODES = ( 'PRIMARY', 'PRIMARY_PREFERRED', 'SECONDARY', 'SECONDARY_PREFERRED', 'NEAREST', ) class ReadPreference(object): """An enum that defines the read preference modes supported by PyMongo. See :doc:`/examples/high_availability` for code examples. A read preference is used in three cases: :class:`~pymongo.mongo_client.MongoClient` connected to a single mongod: - ``PRIMARY``: Queries are allowed if the server is standalone or a replica set primary. - All other modes allow queries to standalone servers, to a replica set primary, or to replica set secondaries. :class:`~pymongo.mongo_client.MongoClient` initialized with the ``replicaSet`` option: - ``PRIMARY``: Read from the primary. This is the default, and provides the strongest consistency. If no primary is available, raise :class:`~pymongo.errors.AutoReconnect`. - ``PRIMARY_PREFERRED``: Read from the primary if available, or if there is none, read from a secondary. - ``SECONDARY``: Read from a secondary. If no secondary is available, raise :class:`~pymongo.errors.AutoReconnect`. - ``SECONDARY_PREFERRED``: Read from a secondary if available, otherwise from the primary. - ``NEAREST``: Read from any member. :class:`~pymongo.mongo_client.MongoClient` connected to a mongos, with a sharded cluster of replica sets: - ``PRIMARY``: Read from the primary of the shard, or raise :class:`~pymongo.errors.OperationFailure` if there is none. This is the default. - ``PRIMARY_PREFERRED``: Read from the primary of the shard, or if there is none, read from a secondary of the shard. - ``SECONDARY``: Read from a secondary of the shard, or raise :class:`~pymongo.errors.OperationFailure` if there is none. - ``SECONDARY_PREFERRED``: Read from a secondary of the shard if available, otherwise from the shard primary. - ``NEAREST``: Read from any shard member. """ PRIMARY = Primary() PRIMARY_PREFERRED = PrimaryPreferred() SECONDARY = Secondary() SECONDARY_PREFERRED = SecondaryPreferred() NEAREST = Nearest() def read_pref_mode_from_name(name): """Get the read preference mode from mongos/uri name. """ return _MONGOS_MODES.index(name) class MovingAverage(object): """Tracks an exponentially-weighted moving average.""" def __init__(self): self.average = None def add_sample(self, sample): if sample < 0: # Likely system time change while waiting for ismaster response # and not using time.monotonic. Ignore it, the next one will # probably be valid. return if self.average is None: self.average = sample else: # The Server Selection Spec requires an exponentially weighted # average with alpha = 0.2. self.average = 0.8 * self.average + 0.2 * sample def get(self): """Get the calculated average, or None if no samples yet.""" return self.average def reset(self): self.average = None pymongo-3.6.1/pymongo/server_type.py0000644000076600000240000000156213245617773020067 0ustar shanestaff00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Type codes for MongoDB servers.""" from collections import namedtuple SERVER_TYPE = namedtuple('ServerType', ['Unknown', 'Mongos', 'RSPrimary', 'RSSecondary', 'RSArbiter', 'RSOther', 'RSGhost', 'Standalone'])(*range(8)) pymongo-3.6.1/pymongo/pool.py0000644000076600000240000011514713245621354016464 0ustar shanestaff00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. import contextlib import os import platform import socket import sys import threading try: import ssl from ssl import SSLError _HAVE_SNI = getattr(ssl, 'HAS_SNI', False) except ImportError: _HAVE_SNI = False class SSLError(socket.error): pass from bson import DEFAULT_CODEC_OPTIONS from bson.py3compat import imap, itervalues, _unicode, integer_types from bson.son import SON from pymongo import auth, helpers, thread_util, __version__ from pymongo.common import (MAX_BSON_SIZE, MAX_MESSAGE_SIZE, MAX_WIRE_VERSION, MAX_WRITE_BATCH_SIZE, ORDERED_TYPES) from pymongo.errors import (AutoReconnect, ConnectionFailure, ConfigurationError, InvalidOperation, DocumentTooLarge, NetworkTimeout, NotMasterError, OperationFailure) from pymongo.ismaster import IsMaster from pymongo.monotonic import time as _time from pymongo.network import (command, receive_message, SocketChecker) from pymongo.read_preferences import ReadPreference from pymongo.server_type import SERVER_TYPE # Always use our backport so we always have support for IP address matching from pymongo.ssl_match_hostname import match_hostname, CertificateError # For SNI support. According to RFC6066, section 3, IPv4 and IPv6 literals are # not permitted for SNI hostname. try: from ipaddress import ip_address def is_ip_address(address): try: ip_address(_unicode(address)) return True except (ValueError, UnicodeError): return False except ImportError: if hasattr(socket, 'inet_pton') and socket.has_ipv6: # Most *nix, recent Windows def is_ip_address(address): try: # inet_pton rejects IPv4 literals with leading zeros # (e.g. 192.168.0.01), inet_aton does not, and we # can connect to them without issue. Use inet_aton. socket.inet_aton(address) return True except socket.error: try: socket.inet_pton(socket.AF_INET6, address) return True except socket.error: return False else: # No inet_pton def is_ip_address(address): try: socket.inet_aton(address) return True except socket.error: if ':' in address: # ':' is not a valid character for a hostname. If we get # here a few things have to be true: # - We're on a recent version of python 2.7 (2.7.9+). # 2.6 and older 2.7 versions don't support SNI. # - We're on Windows XP or some unusual Unix that doesn't # have inet_pton. # - The application is using IPv6 literals with TLS, which # is pretty unusual. return True return False try: from fcntl import fcntl, F_GETFD, F_SETFD, FD_CLOEXEC def _set_non_inheritable_non_atomic(fd): """Set the close-on-exec flag on the given file descriptor.""" flags = fcntl(fd, F_GETFD) fcntl(fd, F_SETFD, flags | FD_CLOEXEC) except ImportError: # Windows, various platforms we don't claim to support # (Jython, IronPython, ...), systems that don't provide # everything we need from fcntl, etc. def _set_non_inheritable_non_atomic(dummy): """Dummy function for platforms that don't provide fcntl.""" pass _MAX_TCP_KEEPIDLE = 300 _MAX_TCP_KEEPINTVL = 10 _MAX_TCP_KEEPCNT = 9 if sys.platform == 'win32': try: import _winreg as winreg except ImportError: import winreg try: with winreg.OpenKey( winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Services\Tcpip\Parameters") as key: _DEFAULT_TCP_IDLE_MS, _ = winreg.QueryValueEx(key, "KeepAliveTime") _DEFAULT_TCP_INTERVAL_MS, _ = winreg.QueryValueEx( key, "KeepAliveInterval") # Make sure these are integers. if not isinstance(_DEFAULT_TCP_IDLE_MS, integer_types): raise ValueError if not isinstance(_DEFAULT_TCP_INTERVAL_MS, integer_types): raise ValueError except (OSError, ValueError): # We could not check the default values so do not attempt to override. def _set_keepalive_times(dummy): pass else: def _set_keepalive_times(sock): idle_ms = min(_DEFAULT_TCP_IDLE_MS, _MAX_TCP_KEEPIDLE * 1000) interval_ms = min(_DEFAULT_TCP_INTERVAL_MS, _MAX_TCP_KEEPINTVL * 1000) if (idle_ms < _DEFAULT_TCP_IDLE_MS or interval_ms < _DEFAULT_TCP_INTERVAL_MS): sock.ioctl(socket.SIO_KEEPALIVE_VALS, (1, idle_ms, interval_ms)) else: def _set_tcp_option(sock, tcp_option, max_value): if hasattr(socket, tcp_option): sockopt = getattr(socket, tcp_option) try: # PYTHON-1350 - NetBSD doesn't implement getsockopt for # TCP_KEEPIDLE and friends. Don't attempt to set the # values there. default = sock.getsockopt(socket.IPPROTO_TCP, sockopt) if default > max_value: sock.setsockopt(socket.IPPROTO_TCP, sockopt, max_value) except socket.error: pass def _set_keepalive_times(sock): _set_tcp_option(sock, 'TCP_KEEPIDLE', _MAX_TCP_KEEPIDLE) _set_tcp_option(sock, 'TCP_KEEPINTVL', _MAX_TCP_KEEPINTVL) _set_tcp_option(sock, 'TCP_KEEPCNT', _MAX_TCP_KEEPCNT) _METADATA = SON([ ('driver', SON([('name', 'PyMongo'), ('version', __version__)])), ]) if sys.platform.startswith('linux'): # platform.linux_distribution was deprecated in Python 3.5. if sys.version_info[:2] < (3, 5): # Distro name and version (e.g. Ubuntu 16.04 xenial) _name = ' '.join([part for part in platform.linux_distribution() if part]) else: _name = platform.system() _METADATA['os'] = SON([ ('type', platform.system()), ('name', _name), ('architecture', platform.machine()), # Kernel version (e.g. 4.4.0-17-generic). ('version', platform.release()) ]) elif sys.platform == 'darwin': _METADATA['os'] = SON([ ('type', platform.system()), ('name', platform.system()), ('architecture', platform.machine()), # (mac|i|tv)OS(X) version (e.g. 10.11.6) instead of darwin # kernel version. ('version', platform.mac_ver()[0]) ]) elif sys.platform == 'win32': _METADATA['os'] = SON([ ('type', platform.system()), # "Windows XP", "Windows 7", "Windows 10", etc. ('name', ' '.join((platform.system(), platform.release()))), ('architecture', platform.machine()), # Windows patch level (e.g. 5.1.2600-SP3) ('version', '-'.join(platform.win32_ver()[1:3])) ]) elif sys.platform.startswith('java'): _name, _ver, _arch = platform.java_ver()[-1] _METADATA['os'] = SON([ # Linux, Windows 7, Mac OS X, etc. ('type', _name), ('name', _name), # x86, x86_64, AMD64, etc. ('architecture', _arch), # Linux kernel version, OSX version, etc. ('version', _ver) ]) else: # Get potential alias (e.g. SunOS 5.11 becomes Solaris 2.11) _aliased = platform.system_alias( platform.system(), platform.release(), platform.version()) _METADATA['os'] = SON([ ('type', platform.system()), ('name', ' '.join([part for part in _aliased[:2] if part])), ('architecture', platform.machine()), ('version', _aliased[2]) ]) if platform.python_implementation().startswith('PyPy'): _METADATA['platform'] = ' '.join( (platform.python_implementation(), '.'.join(imap(str, sys.pypy_version_info)), '(Python %s)' % '.'.join(imap(str, sys.version_info)))) elif sys.platform.startswith('java'): _METADATA['platform'] = ' '.join( (platform.python_implementation(), '.'.join(imap(str, sys.version_info)), '(%s)' % ' '.join((platform.system(), platform.release())))) else: _METADATA['platform'] = ' '.join( (platform.python_implementation(), '.'.join(imap(str, sys.version_info)))) # If the first getaddrinfo call of this interpreter's life is on a thread, # while the main thread holds the import lock, getaddrinfo deadlocks trying # to import the IDNA codec. Import it here, where presumably we're on the # main thread, to avoid the deadlock. See PYTHON-607. u'foo'.encode('idna') def _raise_connection_failure(address, error): """Convert a socket.error to ConnectionFailure and raise it.""" host, port = address # If connecting to a Unix socket, port will be None. if port is not None: msg = '%s:%d: %s' % (host, port, error) else: msg = '%s: %s' % (host, error) if isinstance(error, socket.timeout): raise NetworkTimeout(msg) elif isinstance(error, SSLError) and 'timed out' in str(error): # CPython 2.6, 2.7, PyPy 2.x, and PyPy3 do not distinguish network # timeouts from other SSLErrors (https://bugs.python.org/issue10272). # Luckily, we can work around this limitation because the phrase # 'timed out' appears in all the timeout related SSLErrors raised # on the above platforms. CPython >= 3.2 and PyPy3.3 correctly raise # socket.timeout. raise NetworkTimeout(msg) else: raise AutoReconnect(msg) class PoolOptions(object): __slots__ = ('__max_pool_size', '__min_pool_size', '__max_idle_time_seconds', '__connect_timeout', '__socket_timeout', '__wait_queue_timeout', '__wait_queue_multiple', '__ssl_context', '__ssl_match_hostname', '__socket_keepalive', '__event_listeners', '__appname', '__metadata') def __init__(self, max_pool_size=100, min_pool_size=0, max_idle_time_seconds=None, connect_timeout=None, socket_timeout=None, wait_queue_timeout=None, wait_queue_multiple=None, ssl_context=None, ssl_match_hostname=True, socket_keepalive=True, event_listeners=None, appname=None): self.__max_pool_size = max_pool_size self.__min_pool_size = min_pool_size self.__max_idle_time_seconds = max_idle_time_seconds self.__connect_timeout = connect_timeout self.__socket_timeout = socket_timeout self.__wait_queue_timeout = wait_queue_timeout self.__wait_queue_multiple = wait_queue_multiple self.__ssl_context = ssl_context self.__ssl_match_hostname = ssl_match_hostname self.__socket_keepalive = socket_keepalive self.__event_listeners = event_listeners self.__appname = appname self.__metadata = _METADATA.copy() if appname: self.__metadata['application'] = {'name': appname} @property def max_pool_size(self): """The maximum allowable number of concurrent connections to each connected server. Requests to a server will block if there are `maxPoolSize` outstanding connections to the requested server. Defaults to 100. Cannot be 0. When a server's pool has reached `max_pool_size`, operations for that server block waiting for a socket to be returned to the pool. If ``waitQueueTimeoutMS`` is set, a blocked operation will raise :exc:`~pymongo.errors.ConnectionFailure` after a timeout. By default ``waitQueueTimeoutMS`` is not set. """ return self.__max_pool_size @property def min_pool_size(self): """The minimum required number of concurrent connections that the pool will maintain to each connected server. Default is 0. """ return self.__min_pool_size @property def max_idle_time_seconds(self): """The maximum number of seconds that a connection can remain idle in the pool before being removed and replaced. Defaults to `None` (no limit). """ return self.__max_idle_time_seconds @property def connect_timeout(self): """How long a connection can take to be opened before timing out. """ return self.__connect_timeout @property def socket_timeout(self): """How long a send or receive on a socket can take before timing out. """ return self.__socket_timeout @property def wait_queue_timeout(self): """How long a thread will wait for a socket from the pool if the pool has no free sockets. """ return self.__wait_queue_timeout @property def wait_queue_multiple(self): """Multiplied by max_pool_size to give the number of threads allowed to wait for a socket at one time. """ return self.__wait_queue_multiple @property def ssl_context(self): """An SSLContext instance or None. """ return self.__ssl_context @property def ssl_match_hostname(self): """Call ssl.match_hostname if cert_reqs is not ssl.CERT_NONE. """ return self.__ssl_match_hostname @property def socket_keepalive(self): """Whether to send periodic messages to determine if a connection is closed. """ return self.__socket_keepalive @property def event_listeners(self): """An instance of pymongo.monitoring._EventListeners. """ return self.__event_listeners @property def appname(self): """The application name, for sending with ismaster in server handshake. """ return self.__appname @property def metadata(self): """A dict of metadata about the application, driver, os, and platform. """ return self.__metadata.copy() class SocketInfo(object): """Store a socket with some metadata. :Parameters: - `sock`: a raw socket object - `pool`: a Pool instance - `address`: the server's (host, port) """ def __init__(self, sock, pool, address): self.sock = sock self.address = address self.authset = set() self.closed = False self.last_checkin_time = _time() self.performed_handshake = False self.is_writable = False self.max_wire_version = MAX_WIRE_VERSION self.max_bson_size = MAX_BSON_SIZE self.max_message_size = MAX_MESSAGE_SIZE self.max_write_batch_size = MAX_WRITE_BATCH_SIZE self.supports_sessions = False self.is_mongos = False self.listeners = pool.opts.event_listeners # The pool's pool_id changes with each reset() so we can close sockets # created before the last reset. self.pool_id = pool.pool_id def ismaster(self, metadata, cluster_time): cmd = SON([('ismaster', 1)]) if not self.performed_handshake: cmd['client'] = metadata self.performed_handshake = True if self.max_wire_version >= 6 and cluster_time is not None: cmd['$clusterTime'] = cluster_time ismaster = IsMaster(self.command('admin', cmd, publish_events=False)) self.is_writable = ismaster.is_writable self.max_wire_version = ismaster.max_wire_version self.max_bson_size = ismaster.max_bson_size self.max_message_size = ismaster.max_message_size self.max_write_batch_size = ismaster.max_write_batch_size self.supports_sessions = ( ismaster.logical_session_timeout_minutes is not None) self.is_mongos = ismaster.server_type == SERVER_TYPE.Mongos return ismaster def command(self, dbname, spec, slave_ok=False, read_preference=ReadPreference.PRIMARY, codec_options=DEFAULT_CODEC_OPTIONS, check=True, allowable_errors=None, check_keys=False, read_concern=None, write_concern=None, parse_write_concern_error=False, collation=None, session=None, client=None, retryable_write=False, publish_events=True): """Execute a command or raise an error. :Parameters: - `dbname`: name of the database on which to run the command - `spec`: a command document as a dict, SON, or mapping object - `slave_ok`: whether to set the SlaveOkay wire protocol bit - `read_preference`: a read preference - `codec_options`: a CodecOptions instance - `check`: raise OperationFailure if there are errors - `allowable_errors`: errors to ignore if `check` is True - `check_keys`: if True, check `spec` for invalid keys - `read_concern`: The read concern for this command. - `write_concern`: The write concern for this command. - `parse_write_concern_error`: Whether to parse the ``writeConcernError`` field in the command response. - `collation`: The collation for this command. - `session`: optional ClientSession instance. - `client`: optional MongoClient for gossipping $clusterTime. - `retryable_write`: True if this command is a retryable write. - `publish_events`: Should we publish events for this command? """ self.validate_session(client, session) if (read_concern and self.max_wire_version < 4 and not read_concern.ok_for_legacy): raise ConfigurationError( 'read concern level of %s is not valid ' 'with a max wire version of %d.' % (read_concern.level, self.max_wire_version)) if not (write_concern is None or write_concern.acknowledged or collation is None): raise ConfigurationError( 'Collation is unsupported for unacknowledged writes.') if self.max_wire_version >= 5 and write_concern: spec['writeConcern'] = write_concern.document elif self.max_wire_version < 5 and collation is not None: raise ConfigurationError( 'Must be connected to MongoDB 3.4+ to use a collation.') if (client or session) and not isinstance(spec, ORDERED_TYPES): # Ensure command name remains in first place. spec = SON(spec) if session: spec['lsid'] = session._use_lsid() if retryable_write: spec['txnNumber'] = session._transaction_id() self.send_cluster_time(spec, session, client) listeners = self.listeners if publish_events else None try: return command(self.sock, dbname, spec, slave_ok, self.is_mongos, read_preference, codec_options, session, client, check, allowable_errors, self.address, check_keys, listeners, self.max_bson_size, read_concern, parse_write_concern_error=parse_write_concern_error, collation=collation) except OperationFailure: raise # Catch socket.error, KeyboardInterrupt, etc. and close ourselves. except BaseException as error: self._raise_connection_failure(error) def send_message(self, message, max_doc_size): """Send a raw BSON message or raise ConnectionFailure. If a network exception is raised, the socket is closed. """ if (self.max_bson_size is not None and max_doc_size > self.max_bson_size): raise DocumentTooLarge( "BSON document too large (%d bytes) - the connected server " "supports BSON document sizes up to %d bytes." % (max_doc_size, self.max_bson_size)) try: self.sock.sendall(message) except BaseException as error: self._raise_connection_failure(error) def receive_message(self, request_id): """Receive a raw BSON message or raise ConnectionFailure. If any exception is raised, the socket is closed. """ try: return receive_message(self.sock, request_id, self.max_message_size) except BaseException as error: self._raise_connection_failure(error) def legacy_write(self, request_id, msg, max_doc_size, with_last_error): """Send OP_INSERT, etc., optionally returning response as a dict. Can raise ConnectionFailure or OperationFailure. :Parameters: - `request_id`: an int. - `msg`: bytes, an OP_INSERT, OP_UPDATE, or OP_DELETE message, perhaps with a getlasterror command appended. - `max_doc_size`: size in bytes of the largest document in `msg`. - `with_last_error`: True if a getlasterror command is appended. """ if not with_last_error and not self.is_writable: # Write won't succeed, bail as if we'd done a getlasterror. raise NotMasterError("not master") self.send_message(msg, max_doc_size) if with_last_error: reply = self.receive_message(request_id) return helpers._check_gle_response(reply.command_response()) def write_command(self, request_id, msg): """Send "insert" etc. command, returning response as a dict. Can raise ConnectionFailure or OperationFailure. :Parameters: - `request_id`: an int. - `msg`: bytes, the command message. """ self.send_message(msg, 0) reply = self.receive_message(request_id) result = reply.command_response() # Raises NotMasterError or OperationFailure. helpers._check_command_response(result) return result def check_auth(self, all_credentials): """Update this socket's authentication. Log in or out to bring this socket's credentials up to date with those provided. Can raise ConnectionFailure or OperationFailure. :Parameters: - `all_credentials`: dict, maps auth source to MongoCredential. """ if all_credentials or self.authset: cached = set(itervalues(all_credentials)) authset = self.authset.copy() # Logout any credentials that no longer exist in the cache. for credentials in authset - cached: auth.logout(credentials.source, self) self.authset.discard(credentials) for credentials in cached - authset: auth.authenticate(credentials, self) self.authset.add(credentials) def authenticate(self, credentials): """Log in to the server and store these credentials in `authset`. Can raise ConnectionFailure or OperationFailure. :Parameters: - `credentials`: A MongoCredential. """ auth.authenticate(credentials, self) self.authset.add(credentials) def validate_session(self, client, session): """Validate this session before use with client. Raises error if this session is logged in as a different user or the client is not the one that created the session. """ if session: if session._client is not client: raise InvalidOperation( 'Can only use session with the MongoClient that' ' started it') if session._authset != self.authset: raise InvalidOperation( 'Cannot use session after authenticating with different' ' credentials') def close(self): self.closed = True # Avoid exceptions on interpreter shutdown. try: self.sock.close() except Exception: pass def send_cluster_time(self, command, session, client): """Add cluster time for MongoDB >= 3.6.""" if self.max_wire_version >= 6 and client: client._send_cluster_time(command, session) def update_last_checkin_time(self): self.last_checkin_time = _time() def idle_time_seconds(self): """Seconds since this socket was last checked into its pool.""" return _time() - self.last_checkin_time def _raise_connection_failure(self, error): # Catch *all* exceptions from socket methods and close the socket. In # regular Python, socket operations only raise socket.error, even if # the underlying cause was a Ctrl-C: a signal raised during socket.recv # is expressed as an EINTR error from poll. See internal_select_ex() in # socketmodule.c. All error codes from poll become socket.error at # first. Eventually in PyEval_EvalFrameEx the interpreter checks for # signals and throws KeyboardInterrupt into the current frame on the # main thread. # # But in Gevent and Eventlet, the polling mechanism (epoll, kqueue, # ...) is called in Python code, which experiences the signal as a # KeyboardInterrupt from the start, rather than as an initial # socket.error, so we catch that, close the socket, and reraise it. self.close() if isinstance(error, socket.error): _raise_connection_failure(self.address, error) else: raise error def __eq__(self, other): return self.sock == other.sock def __ne__(self, other): return not self == other def __hash__(self): return hash(self.sock) def __repr__(self): return "SocketInfo(%s)%s at %s" % ( repr(self.sock), self.closed and " CLOSED" or "", id(self) ) def _create_connection(address, options): """Given (host, port) and PoolOptions, connect and return a socket object. Can raise socket.error. This is a modified version of create_connection from CPython >= 2.6. """ host, port = address # Check if dealing with a unix domain socket if host.endswith('.sock'): if not hasattr(socket, "AF_UNIX"): raise ConnectionFailure("UNIX-sockets are not supported " "on this system") sock = socket.socket(socket.AF_UNIX) # SOCK_CLOEXEC not supported for Unix sockets. _set_non_inheritable_non_atomic(sock.fileno()) try: sock.connect(host) return sock except socket.error: sock.close() raise # Don't try IPv6 if we don't support it. Also skip it if host # is 'localhost' (::1 is fine). Avoids slow connect issues # like PYTHON-356. family = socket.AF_INET if socket.has_ipv6 and host != 'localhost': family = socket.AF_UNSPEC err = None for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): af, socktype, proto, dummy, sa = res # SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited # number of platforms (newer Linux and *BSD). Starting with CPython 3.4 # all file descriptors are created non-inheritable. See PEP 446. try: sock = socket.socket( af, socktype | getattr(socket, 'SOCK_CLOEXEC', 0), proto) except socket.error: # Can SOCK_CLOEXEC be defined even if the kernel doesn't support # it? sock = socket.socket(af, socktype, proto) # Fallback when SOCK_CLOEXEC isn't available. _set_non_inheritable_non_atomic(sock.fileno()) try: sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) sock.settimeout(options.connect_timeout) sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, options.socket_keepalive) if options.socket_keepalive: _set_keepalive_times(sock) sock.connect(sa) return sock except socket.error as e: err = e sock.close() if err is not None: raise err else: # This likely means we tried to connect to an IPv6 only # host with an OS/kernel or Python interpreter that doesn't # support IPv6. The test case is Jython2.5.1 which doesn't # support IPv6 at all. raise socket.error('getaddrinfo failed') def _configured_socket(address, options): """Given (host, port) and PoolOptions, return a configured socket. Can raise socket.error, ConnectionFailure, or CertificateError. Sets socket's SSL and timeout options. """ sock = _create_connection(address, options) ssl_context = options.ssl_context if ssl_context is not None: host = address[0] try: # According to RFC6066, section 3, IPv4 and IPv6 literals are # not permitted for SNI hostname. if _HAVE_SNI and not is_ip_address(host): sock = ssl_context.wrap_socket(sock, server_hostname=host) else: sock = ssl_context.wrap_socket(sock) except IOError as exc: sock.close() raise ConnectionFailure("SSL handshake failed: %s" % (str(exc),)) if ssl_context.verify_mode and options.ssl_match_hostname: try: match_hostname(sock.getpeercert(), hostname=host) except CertificateError: sock.close() raise sock.settimeout(options.socket_timeout) return sock # Do *not* explicitly inherit from object or Jython won't call __del__ # http://bugs.jython.org/issue1057 class Pool: def __init__(self, address, options, handshake=True): """ :Parameters: - `address`: a (hostname, port) tuple - `options`: a PoolOptions instance - `handshake`: whether to call ismaster for each new SocketInfo """ # Check a socket's health with socket_closed() every once in a while. # Can override for testing: 0 to always check, None to never check. self._check_interval_seconds = 1 self.sockets = set() self.lock = threading.Lock() self.active_sockets = 0 # Keep track of resets, so we notice sockets created before the most # recent reset and close them. self.pool_id = 0 self.pid = os.getpid() self.address = address self.opts = options self.handshake = handshake if (self.opts.wait_queue_multiple is None or self.opts.max_pool_size is None): max_waiters = None else: max_waiters = ( self.opts.max_pool_size * self.opts.wait_queue_multiple) self._socket_semaphore = thread_util.create_semaphore( self.opts.max_pool_size, max_waiters) self.socket_checker = SocketChecker() def reset(self): with self.lock: self.pool_id += 1 self.pid = os.getpid() sockets, self.sockets = self.sockets, set() self.active_sockets = 0 for sock_info in sockets: sock_info.close() def remove_stale_sockets(self): """Removes stale sockets then adds new ones if pool is too small.""" if self.opts.max_idle_time_seconds is not None: with self.lock: for sock_info in self.sockets.copy(): if (sock_info.idle_time_seconds() > self.opts.max_idle_time_seconds): self.sockets.remove(sock_info) sock_info.close() while True: with self.lock: if (len(self.sockets) + self.active_sockets >= self.opts.min_pool_size): # There are enough sockets in the pool. break # We must acquire the semaphore to respect max_pool_size. if not self._socket_semaphore.acquire(False): break try: sock_info = self.connect() with self.lock: self.sockets.add(sock_info) finally: self._socket_semaphore.release() def connect(self): """Connect to Mongo and return a new SocketInfo. Can raise ConnectionFailure or CertificateError. Note that the pool does not keep a reference to the socket -- you must call return_socket() when you're done with it. """ sock = None try: sock = _configured_socket(self.address, self.opts) except socket.error as error: if sock is not None: sock.close() _raise_connection_failure(self.address, error) sock_info = SocketInfo(sock, self, self.address) if self.handshake: sock_info.ismaster(self.opts.metadata, None) return sock_info @contextlib.contextmanager def get_socket(self, all_credentials, checkout=False): """Get a socket from the pool. Use with a "with" statement. Returns a :class:`SocketInfo` object wrapping a connected :class:`socket.socket`. This method should always be used in a with-statement:: with pool.get_socket(credentials, checkout) as socket_info: socket_info.send_message(msg) data = socket_info.receive_message(op_code, request_id) The socket is logged in or out as needed to match ``all_credentials`` using the correct authentication mechanism for the server's wire protocol version. Can raise ConnectionFailure or OperationFailure. :Parameters: - `all_credentials`: dict, maps auth source to MongoCredential. - `checkout` (optional): keep socket checked out. """ # First get a socket, then attempt authentication. Simplifies # semaphore management in the face of network errors during auth. sock_info = self._get_socket_no_auth() try: sock_info.check_auth(all_credentials) yield sock_info except: # Exception in caller. Decrement semaphore. self.return_socket(sock_info) raise else: if not checkout: self.return_socket(sock_info) def _get_socket_no_auth(self): """Get or create a SocketInfo. Can raise ConnectionFailure.""" # We use the pid here to avoid issues with fork / multiprocessing. # See test.test_client:TestClient.test_fork for an example of # what could go wrong otherwise if self.pid != os.getpid(): self.reset() # Get a free socket or create one. if not self._socket_semaphore.acquire( True, self.opts.wait_queue_timeout): self._raise_wait_queue_timeout() with self.lock: self.active_sockets += 1 # We've now acquired the semaphore and must release it on error. try: try: # set.pop() isn't atomic in Jython less than 2.7, see # http://bugs.jython.org/issue1854 with self.lock: # Can raise ConnectionFailure. sock_info = self.sockets.pop() except KeyError: # Can raise ConnectionFailure or CertificateError. sock_info = self.connect() else: # Can raise ConnectionFailure. sock_info = self._check(sock_info) except: self._socket_semaphore.release() with self.lock: self.active_sockets -= 1 raise return sock_info def return_socket(self, sock_info): """Return the socket to the pool, or if it's closed discard it.""" if self.pid != os.getpid(): self.reset() else: if sock_info.pool_id != self.pool_id: sock_info.close() elif not sock_info.closed: sock_info.update_last_checkin_time() with self.lock: self.sockets.add(sock_info) self._socket_semaphore.release() with self.lock: self.active_sockets -= 1 def _check(self, sock_info): """This side-effecty function checks if this socket has been idle for for longer than the max idle time, or if the socket has been closed by some external network error, and if so, attempts to create a new socket. If this connection attempt fails we raise the ConnectionFailure. Checking sockets lets us avoid seeing *some* :class:`~pymongo.errors.AutoReconnect` exceptions on server hiccups, etc. We only check if the socket was closed by an external error if it has been > 1 second since the socket was checked into the pool, to keep performance reasonable - we can't avoid AutoReconnects completely anyway. """ idle_time_seconds = sock_info.idle_time_seconds() # If socket is idle, open a new one. if (self.opts.max_idle_time_seconds is not None and idle_time_seconds > self.opts.max_idle_time_seconds): sock_info.close() return self.connect() if (self._check_interval_seconds is not None and ( 0 == self._check_interval_seconds or idle_time_seconds > self._check_interval_seconds)): if self.socket_checker.socket_closed(sock_info.sock): sock_info.close() return self.connect() return sock_info def _raise_wait_queue_timeout(self): raise ConnectionFailure( 'Timed out waiting for socket from pool with max_size %r and' ' wait_queue_timeout %r' % ( self.opts.max_pool_size, self.opts.wait_queue_timeout)) def __del__(self): # Avoid ResourceWarnings in Python 3 for sock_info in self.sockets: sock_info.close() pymongo-3.6.1/pymongo/thread_util.py0000644000076600000240000000756713245617773020037 0ustar shanestaff00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for multi-threading support.""" import threading try: from time import monotonic as _time except ImportError: from time import time as _time from pymongo.monotonic import time as _time from pymongo.errors import ExceededMaxWaiters ### Begin backport from CPython 3.2 for timeout support for Semaphore.acquire class Semaphore: # After Tim Peters' semaphore class, but not quite the same (no maximum) def __init__(self, value=1): if value < 0: raise ValueError("semaphore initial value must be >= 0") self._cond = threading.Condition(threading.Lock()) self._value = value def acquire(self, blocking=True, timeout=None): if not blocking and timeout is not None: raise ValueError("can't specify timeout for non-blocking acquire") rc = False endtime = None self._cond.acquire() while self._value == 0: if not blocking: break if timeout is not None: if endtime is None: endtime = _time() + timeout else: timeout = endtime - _time() if timeout <= 0: break self._cond.wait(timeout) else: self._value = self._value - 1 rc = True self._cond.release() return rc __enter__ = acquire def release(self): self._cond.acquire() self._value = self._value + 1 self._cond.notify() self._cond.release() def __exit__(self, t, v, tb): self.release() @property def counter(self): return self._value class BoundedSemaphore(Semaphore): """Semaphore that checks that # releases is <= # acquires""" def __init__(self, value=1): Semaphore.__init__(self, value) self._initial_value = value def release(self): if self._value >= self._initial_value: raise ValueError("Semaphore released too many times") return Semaphore.release(self) ### End backport from CPython 3.2 class DummySemaphore(object): def __init__(self, value=None): pass def acquire(self, blocking=True, timeout=None): return True def release(self): pass class MaxWaitersBoundedSemaphore(object): def __init__(self, semaphore_class, value=1, max_waiters=1): self.waiter_semaphore = semaphore_class(max_waiters) self.semaphore = semaphore_class(value) def acquire(self, blocking=True, timeout=None): if not self.waiter_semaphore.acquire(False): raise ExceededMaxWaiters() try: return self.semaphore.acquire(blocking, timeout) finally: self.waiter_semaphore.release() def __getattr__(self, name): return getattr(self.semaphore, name) class MaxWaitersBoundedSemaphoreThread(MaxWaitersBoundedSemaphore): def __init__(self, value=1, max_waiters=1): MaxWaitersBoundedSemaphore.__init__( self, BoundedSemaphore, value, max_waiters) def create_semaphore(max_size, max_waiters): if max_size is None: return DummySemaphore() else: if max_waiters is None: return BoundedSemaphore(max_size) else: return MaxWaitersBoundedSemaphoreThread(max_size, max_waiters) pymongo-3.6.1/pymongo/helpers.py0000644000076600000240000002300213245621354017141 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Bits and pieces used by the driver that don't really fit elsewhere.""" import sys import traceback from bson.py3compat import abc, iteritems, itervalues, string_type from bson.son import SON from pymongo import ASCENDING from pymongo.errors import (CursorNotFound, DuplicateKeyError, ExecutionTimeout, NotMasterError, OperationFailure, WriteError, WriteConcernError, WTimeoutError) _UUNDER = u"_" def _gen_index_name(keys): """Generate an index name from the set of fields it is over.""" return _UUNDER.join(["%s_%s" % item for item in keys]) def _index_list(key_or_list, direction=None): """Helper to generate a list of (key, direction) pairs. Takes such a list, or a single key, or a single key and direction. """ if direction is not None: return [(key_or_list, direction)] else: if isinstance(key_or_list, string_type): return [(key_or_list, ASCENDING)] elif not isinstance(key_or_list, (list, tuple)): raise TypeError("if no direction is specified, " "key_or_list must be an instance of list") return key_or_list def _index_document(index_list): """Helper to generate an index specifying document. Takes a list of (key, direction) pairs. """ if isinstance(index_list, abc.Mapping): raise TypeError("passing a dict to sort/create_index/hint is not " "allowed - use a list of tuples instead. did you " "mean %r?" % list(iteritems(index_list))) elif not isinstance(index_list, (list, tuple)): raise TypeError("must use a list of (key, direction) pairs, " "not: " + repr(index_list)) if not len(index_list): raise ValueError("key_or_list must not be the empty list") index = SON() for (key, value) in index_list: if not isinstance(key, string_type): raise TypeError("first item in each key pair must be a string") if not isinstance(value, (string_type, int, abc.Mapping)): raise TypeError("second item in each key pair must be 1, -1, " "'2d', 'geoHaystack', or another valid MongoDB " "index specifier.") index[key] = value return index def _check_command_response(response, msg=None, allowable_errors=None, parse_write_concern_error=False): """Check the response to a command for errors. """ if "ok" not in response: # Server didn't recognize our message as a command. raise OperationFailure(response.get("$err"), response.get("code"), response) # TODO: remove, this is moving to _check_gle_response if response.get("wtimeout", False): # MongoDB versions before 1.8.0 return the error message in an "errmsg" # field. If "errmsg" exists "err" will also exist set to None, so we # have to check for "errmsg" first. raise WTimeoutError(response.get("errmsg", response.get("err")), response.get("code"), response) if parse_write_concern_error and 'writeConcernError' in response: wce = response['writeConcernError'] raise WriteConcernError(wce['errmsg'], wce['code'], wce) if not response["ok"]: details = response # Mongos returns the error details in a 'raw' object # for some errors. if "raw" in response: for shard in itervalues(response["raw"]): # Grab the first non-empty raw error from a shard. if shard.get("errmsg") and not shard.get("ok"): details = shard break errmsg = details["errmsg"] if allowable_errors is None or errmsg not in allowable_errors: # Server is "not master" or "recovering" if (errmsg.startswith("not master") or errmsg.startswith("node is recovering")): raise NotMasterError(errmsg, response) # Server assertion failures if errmsg == "db assertion failure": errmsg = ("db assertion failure, assertion: '%s'" % details.get("assertion", "")) raise OperationFailure(errmsg, details.get("assertionCode"), response) # Other errors code = details.get("code") # findAndModify with upsert can raise duplicate key error if code in (11000, 11001, 12582): raise DuplicateKeyError(errmsg, code, response) elif code == 50: raise ExecutionTimeout(errmsg, code, response) elif code == 43: raise CursorNotFound(errmsg, code, response) msg = msg or "%s" raise OperationFailure(msg % errmsg, code, response) def _check_gle_response(result): """Return getlasterror response as a dict, or raise OperationFailure.""" # Did getlasterror itself fail? _check_command_response(result) if result.get("wtimeout", False): # MongoDB versions before 1.8.0 return the error message in an "errmsg" # field. If "errmsg" exists "err" will also exist set to None, so we # have to check for "errmsg" first. raise WTimeoutError(result.get("errmsg", result.get("err")), result.get("code"), result) error_msg = result.get("err", "") if error_msg is None: return result if error_msg.startswith("not master"): raise NotMasterError(error_msg, result) details = result # mongos returns the error code in an error object for some errors. if "errObjects" in result: for errobj in result["errObjects"]: if errobj.get("err") == error_msg: details = errobj break code = details.get("code") if code in (11000, 11001, 12582): raise DuplicateKeyError(details["err"], code, result) raise OperationFailure(details["err"], code, result) def _raise_last_write_error(write_errors): # If the last batch had multiple errors only report # the last error to emulate continue_on_error. error = write_errors[-1] if error.get("code") == 11000: raise DuplicateKeyError(error.get("errmsg"), 11000, error) raise WriteError(error.get("errmsg"), error.get("code"), error) def _raise_write_concern_error(error): if "errInfo" in error and error["errInfo"].get('wtimeout'): # Make sure we raise WTimeoutError raise WTimeoutError( error.get("errmsg"), error.get("code"), error) raise WriteConcernError( error.get("errmsg"), error.get("code"), error) def _check_write_command_response(result): """Backward compatibility helper for write command error handling. """ # Prefer write errors over write concern errors write_errors = result.get("writeErrors") if write_errors: _raise_last_write_error(write_errors) error = result.get("writeConcernError") if error: _raise_write_concern_error(error) def _raise_last_error(bulk_write_result): """Backward compatibility helper for insert error handling. """ # Prefer write errors over write concern errors write_errors = bulk_write_result.get("writeErrors") if write_errors: _raise_last_write_error(write_errors) _raise_write_concern_error(bulk_write_result["writeConcernErrors"][-1]) def _fields_list_to_dict(fields, option_name): """Takes a sequence of field names and returns a matching dictionary. ["a", "b"] becomes {"a": 1, "b": 1} and ["a.b.c", "d", "a.c"] becomes {"a.b.c": 1, "d": 1, "a.c": 1} """ if isinstance(fields, abc.Mapping): return fields if isinstance(fields, (abc.Sequence, abc.Set)): if not all(isinstance(field, string_type) for field in fields): raise TypeError("%s must be a list of key names, each an " "instance of %s" % (option_name, string_type.__name__)) return dict.fromkeys(fields, 1) raise TypeError("%s must be a mapping or " "list of key names" % (option_name,)) def _handle_exception(): """Print exceptions raised by subscribers to stderr.""" # Heavily influenced by logging.Handler.handleError. # See note here: # https://docs.python.org/3.4/library/sys.html#sys.__stderr__ if sys.stderr: einfo = sys.exc_info() try: traceback.print_exception(einfo[0], einfo[1], einfo[2], None, sys.stderr) except IOError: pass finally: del einfo pymongo-3.6.1/pymongo/ssl_match_hostname.py0000644000076600000240000001112013245617773021362 0ustar shanestaff00000000000000# Backport of the match_hostname logic from python 3.5, with small # changes to support IP address matching on python 2.6, 2.7, 3.3, and 3.4. import re import sys try: # Python 3.3+, or the ipaddress module from pypi. from ipaddress import ip_address except ImportError: ip_address = lambda address: None # ipaddress.ip_address requires unicode if sys.version_info[0] < 3: _unicode = unicode else: _unicode = lambda value: value class CertificateError(ValueError): pass def _dnsname_match(dn, hostname, max_wildcards=1): """Matching according to RFC 6125, section 6.4.3 http://tools.ietf.org/html/rfc6125#section-6.4.3 """ pats = [] if not dn: return False parts = dn.split(r'.') leftmost = parts[0] remainder = parts[1:] wildcards = leftmost.count('*') if wildcards > max_wildcards: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a # reasonable choice. raise CertificateError( "too many wildcards in certificate DNS name: " + repr(dn)) # speed up common case w/o wildcards if not wildcards: return dn.lower() == hostname.lower() # RFC 6125, section 6.4.3, subitem 1. # The client SHOULD NOT attempt to match a presented identifier in which # the wildcard character comprises a label other than the left-most label. if leftmost == '*': # When '*' is a fragment by itself, it matches a non-empty dotless # fragment. pats.append('[^.]+') elif leftmost.startswith('xn--') or hostname.startswith('xn--'): # RFC 6125, section 6.4.3, subitem 3. # The client SHOULD NOT attempt to match a presented identifier # where the wildcard character is embedded within an A-label or # U-label of an internationalized domain name. pats.append(re.escape(leftmost)) else: # Otherwise, '*' matches any dotless string, e.g. www* pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) # add the remaining fragments, ignore any wildcards for frag in remainder: pats.append(re.escape(frag)) pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) return pat.match(hostname) def _ipaddress_match(ipname, host_ip): """Exact matching of IP addresses. RFC 6125 explicitly doesn't define an algorithm for this (section 1.7.2 - "Out of Scope"). """ # OpenSSL may add a trailing newline to a subjectAltName's IP address ip = ip_address(_unicode(ipname).rstrip()) return ip == host_ip def match_hostname(cert, hostname): """Verify that *cert* (in decoded format as returned by SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 rules are followed. CertificateError is raised on failure. On success, the function returns nothing. """ if not cert: raise ValueError("empty or no certificate, match_hostname needs a " "SSL socket or SSL context with either " "CERT_OPTIONAL or CERT_REQUIRED") try: host_ip = ip_address(_unicode(hostname)) except (ValueError, UnicodeError): # Not an IP address (common case) host_ip = None dnsnames = [] san = cert.get('subjectAltName', ()) for key, value in san: if key == 'DNS': if host_ip is None and _dnsname_match(value, hostname): return dnsnames.append(value) elif key == 'IP Address': if host_ip is not None and _ipaddress_match(value, host_ip): return dnsnames.append(value) if not dnsnames: # The subject is only checked when there is no dNSName entry # in subjectAltName for sub in cert.get('subject', ()): for key, value in sub: # XXX according to RFC 2818, the most specific Common Name # must be used. if key == 'commonName': if _dnsname_match(value, hostname): return dnsnames.append(value) if len(dnsnames) > 1: raise CertificateError("hostname %r " "doesn't match either of %s" % (hostname, ', '.join(map(repr, dnsnames)))) elif len(dnsnames) == 1: raise CertificateError("hostname %r " "doesn't match %r" % (hostname, dnsnames[0])) else: raise CertificateError("no appropriate commonName or " "subjectAltName fields were found") pymongo-3.6.1/pymongo/client_session.py0000644000076600000240000002230413245621354020524 0ustar shanestaff00000000000000# Copyright 2017 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Logical sessions for ordering sequential operations. Requires MongoDB 3.6. .. versionadded:: 3.6 Causally Consistent Reads ========================= .. code-block:: python with client.start_session(causal_consistency=True) as session: collection = client.db.collection collection.update_one({'_id': 1}, {'$set': {'x': 10}}, session=session) secondary_c = collection.with_options( read_preference=ReadPreference.SECONDARY) # A secondary read waits for replication of the write. secondary_c.find_one({'_id': 1}, session=session) If `causal_consistency` is True (the default), read operations that use the session are causally after previous read and write operations. Using a causally consistent session, an application can read its own writes and is guaranteed monotonic reads, even when reading from replica set secondaries. .. mongodoc:: causal-consistency Classes ======= """ import collections import uuid from bson.binary import Binary from bson.int64 import Int64 from bson.py3compat import abc from bson.timestamp import Timestamp from pymongo import monotonic from pymongo.errors import InvalidOperation class SessionOptions(object): """Options for a new :class:`ClientSession`. :Parameters: - `causal_consistency` (optional): If True (the default), read operations are causally ordered within the session. """ def __init__(self, causal_consistency=True): self._causal_consistency = causal_consistency @property def causal_consistency(self): """Whether causal consistency is configured.""" return self._causal_consistency class ClientSession(object): """A session for ordering sequential operations.""" def __init__(self, client, server_session, options, authset): # A MongoClient, a _ServerSession, a SessionOptions, and a set. self._client = client self._server_session = server_session self._options = options self._authset = authset self._cluster_time = None self._operation_time = None def end_session(self): """Finish this session. It is an error to use the session or any derived :class:`~pymongo.database.Database`, :class:`~pymongo.collection.Collection`, or :class:`~pymongo.cursor.Cursor` after the session has ended. """ self._end_session(True) def _end_session(self, lock): if self._server_session is not None: self._client._return_server_session(self._server_session, lock) self._server_session = None def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.end_session() @property def client(self): """The :class:`~pymongo.mongo_client.MongoClient` this session was created from. """ return self._client @property def options(self): """The :class:`SessionOptions` this session was created with.""" return self._options @property def session_id(self): """A BSON document, the opaque server session identifier.""" if self._server_session is None: raise InvalidOperation("Cannot use ended session") return self._server_session.session_id @property def cluster_time(self): """The cluster time returned by the last operation executed in this session. """ return self._cluster_time @property def operation_time(self): """The operation time returned by the last operation executed in this session. """ return self._operation_time def _advance_cluster_time(self, cluster_time): """Internal cluster time helper.""" if self._cluster_time is None: self._cluster_time = cluster_time elif cluster_time is not None: if cluster_time["clusterTime"] > self._cluster_time["clusterTime"]: self._cluster_time = cluster_time def advance_cluster_time(self, cluster_time): """Update the cluster time for this session. :Parameters: - `cluster_time`: The :data:`~pymongo.client_session.ClientSession.cluster_time` from another `ClientSession` instance. """ if not isinstance(cluster_time, abc.Mapping): raise TypeError( "cluster_time must be a subclass of collections.Mapping") if not isinstance(cluster_time.get("clusterTime"), Timestamp): raise ValueError("Invalid cluster_time") self._advance_cluster_time(cluster_time) def _advance_operation_time(self, operation_time): """Internal operation time helper.""" if self._operation_time is None: self._operation_time = operation_time elif operation_time is not None: if operation_time > self._operation_time: self._operation_time = operation_time def advance_operation_time(self, operation_time): """Update the operation time for this session. :Parameters: - `operation_time`: The :data:`~pymongo.client_session.ClientSession.operation_time` from another `ClientSession` instance. """ if not isinstance(operation_time, Timestamp): raise TypeError("operation_time must be an instance " "of bson.timestamp.Timestamp") self._advance_operation_time(operation_time) @property def has_ended(self): """True if this session is finished.""" return self._server_session is None def _use_lsid(self): # Internal function. if self._server_session is None: raise InvalidOperation("Cannot use ended session") return self._server_session.use_lsid() def _transaction_id(self): # Internal function. if self._server_session is None: raise InvalidOperation("Cannot use ended session") return self._server_session.transaction_id() def _retry_transaction_id(self): # Internal function. if self._server_session is None: raise InvalidOperation("Cannot use ended session") self._server_session.retry_transaction_id() class _ServerSession(object): def __init__(self): # Ensure id is type 4, regardless of CodecOptions.uuid_representation. self.session_id = {'id': Binary(uuid.uuid4().bytes, 4)} self.last_use = monotonic.time() self._transaction_id = 0 def timed_out(self, session_timeout_minutes): idle_seconds = monotonic.time() - self.last_use # Timed out if we have less than a minute to live. return idle_seconds > (session_timeout_minutes - 1) * 60 def use_lsid(self): self.last_use = monotonic.time() return self.session_id def transaction_id(self): """Monotonically increasing positive 64-bit integer.""" self._transaction_id += 1 return Int64(self._transaction_id) def retry_transaction_id(self): self._transaction_id -= 1 class _ServerSessionPool(collections.deque): """Pool of _ServerSession objects. This class is not thread-safe, access it while holding the Topology lock. """ def pop_all(self): ids = [] while self: ids.append(self.pop().session_id) return ids def get_server_session(self, session_timeout_minutes): # Although the Driver Sessions Spec says we only clear stale sessions # in return_server_session, PyMongo can't take a lock when returning # sessions from a __del__ method (like in Cursor.__die), so it can't # clear stale sessions there. In case many sessions were returned via # __del__, check for stale sessions here too. self._clear_stale(session_timeout_minutes) # The most recently used sessions are on the left. while self: s = self.popleft() if not s.timed_out(session_timeout_minutes): return s return _ServerSession() def return_server_session(self, server_session, session_timeout_minutes): self._clear_stale(session_timeout_minutes) if not server_session.timed_out(session_timeout_minutes): self.appendleft(server_session) def return_server_session_no_lock(self, server_session): self.appendleft(server_session) def _clear_stale(self, session_timeout_minutes): # Clear stale sessions. The least recently used are on the right. while self: if self[-1].timed_out(session_timeout_minutes): self.pop() else: # The remaining sessions also haven't timed out. break pymongo-3.6.1/pymongo/cursor.py0000644000076600000240000013414613245621354017030 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Cursor class to iterate over Mongo query results.""" import copy import datetime import warnings from collections import deque from bson import RE_TYPE from bson.code import Code from bson.py3compat import (iteritems, integer_types, string_type) from bson.son import SON from pymongo import helpers from pymongo.common import validate_boolean, validate_is_mapping from pymongo.collation import validate_collation_or_none from pymongo.errors import (AutoReconnect, ConnectionFailure, InvalidOperation, NotMasterError, OperationFailure) from pymongo.message import (_convert_exception, _CursorAddress, _GetMore, _RawBatchGetMore, _Query, _RawBatchQuery) from pymongo.read_preferences import ReadPreference _QUERY_OPTIONS = { "tailable_cursor": 2, "slave_okay": 4, "oplog_replay": 8, "no_timeout": 16, "await_data": 32, "exhaust": 64, "partial": 128} class CursorType(object): NON_TAILABLE = 0 """The standard cursor type.""" TAILABLE = _QUERY_OPTIONS["tailable_cursor"] """The tailable cursor type. Tailable cursors are only for use with capped collections. They are not closed when the last data is retrieved but are kept open and the cursor location marks the final document position. If more data is received iteration of the cursor will continue from the last document received. """ TAILABLE_AWAIT = TAILABLE | _QUERY_OPTIONS["await_data"] """A tailable cursor with the await option set. Creates a tailable cursor that will wait for a few seconds after returning the full result set so that it can capture and return additional data added during the query. """ EXHAUST = _QUERY_OPTIONS["exhaust"] """An exhaust cursor. MongoDB will stream batched results to the client without waiting for the client to request each batch, reducing latency. """ # This has to be an old style class due to # http://bugs.jython.org/issue1057 class _SocketManager: """Used with exhaust cursors to ensure the socket is returned. """ def __init__(self, sock, pool): self.sock = sock self.pool = pool self.__closed = False def __del__(self): self.close() def close(self): """Return this instance's socket to the connection pool. """ if not self.__closed: self.__closed = True self.pool.return_socket(self.sock) self.sock, self.pool = None, None class Cursor(object): """A cursor / iterator over Mongo query results. """ _query_class = _Query _getmore_class = _GetMore def __init__(self, collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None): """Create a new cursor. Should not be called directly by application developers - see :meth:`~pymongo.collection.Collection.find` instead. .. mongodoc:: cursors """ # Initialize all attributes used in __del__ before possibly raising # an error to avoid attribute errors during garbage collection. self.__id = None self.__exhaust = False self.__exhaust_mgr = None self.__killed = False if session: self.__session = session self.__explicit_session = True else: self.__session = None self.__explicit_session = False spec = filter if spec is None: spec = {} validate_is_mapping("filter", spec) if not isinstance(skip, int): raise TypeError("skip must be an instance of int") if not isinstance(limit, int): raise TypeError("limit must be an instance of int") validate_boolean("no_cursor_timeout", no_cursor_timeout) if cursor_type not in (CursorType.NON_TAILABLE, CursorType.TAILABLE, CursorType.TAILABLE_AWAIT, CursorType.EXHAUST): raise ValueError("not a valid value for cursor_type") validate_boolean("allow_partial_results", allow_partial_results) validate_boolean("oplog_replay", oplog_replay) if modifiers is not None: warnings.warn("the 'modifiers' parameter is deprecated", DeprecationWarning, stacklevel=2) validate_is_mapping("modifiers", modifiers) if not isinstance(batch_size, integer_types): raise TypeError("batch_size must be an integer") if batch_size < 0: raise ValueError("batch_size must be >= 0") if projection is not None: if not projection: projection = {"_id": 1} projection = helpers._fields_list_to_dict(projection, "projection") self.__collection = collection self.__spec = spec self.__projection = projection self.__skip = skip self.__limit = limit self.__batch_size = batch_size self.__modifiers = modifiers and modifiers.copy() or {} self.__ordering = sort and helpers._index_document(sort) or None self.__max_scan = max_scan self.__explain = False self.__comment = comment self.__max_time_ms = max_time_ms self.__max_await_time_ms = None self.__max = max self.__min = min self.__manipulate = manipulate self.__collation = validate_collation_or_none(collation) self.__return_key = return_key self.__show_record_id = show_record_id self.__snapshot = snapshot self.__set_hint(hint) # Exhaust cursor support if cursor_type == CursorType.EXHAUST: if self.__collection.database.client.is_mongos: raise InvalidOperation('Exhaust cursors are ' 'not supported by mongos') if limit: raise InvalidOperation("Can't use limit and exhaust together.") self.__exhaust = True # This is ugly. People want to be able to do cursor[5:5] and # get an empty result set (old behavior was an # exception). It's hard to do that right, though, because the # server uses limit(0) to mean 'no limit'. So we set __empty # in that case and check for it when iterating. We also unset # it anytime we change __limit. self.__empty = False self.__data = deque() self.__address = None self.__retrieved = 0 self.__codec_options = collection.codec_options self.__read_preference = collection.read_preference self.__read_concern = collection.read_concern self.__query_flags = cursor_type if self.__read_preference != ReadPreference.PRIMARY: self.__query_flags |= _QUERY_OPTIONS["slave_okay"] if no_cursor_timeout: self.__query_flags |= _QUERY_OPTIONS["no_timeout"] if allow_partial_results: self.__query_flags |= _QUERY_OPTIONS["partial"] if oplog_replay: self.__query_flags |= _QUERY_OPTIONS["oplog_replay"] @property def collection(self): """The :class:`~pymongo.collection.Collection` that this :class:`Cursor` is iterating. """ return self.__collection @property def retrieved(self): """The number of documents retrieved so far. """ return self.__retrieved def __del__(self): self.__die() def rewind(self): """Rewind this cursor to its unevaluated state. Reset this cursor if it has been partially or completely evaluated. Any options that are present on the cursor will remain in effect. Future iterating performed on this cursor will cause new queries to be sent to the server, even if the resultant data has already been retrieved by this cursor. """ self.__data = deque() self.__id = None self.__address = None self.__retrieved = 0 self.__killed = False return self def clone(self): """Get a clone of this cursor. Returns a new Cursor instance with options matching those that have been set on the current instance. The clone will be completely unevaluated, even if the current instance has been partially or completely evaluated. """ return self._clone(True) def _clone(self, deepcopy=True, base=None): """Internal clone helper.""" if not base: if self.__explicit_session: base = self._clone_base(self.__session) else: base = self._clone_base(None) values_to_clone = ("spec", "projection", "skip", "limit", "max_time_ms", "max_await_time_ms", "comment", "max", "min", "ordering", "explain", "hint", "batch_size", "max_scan", "manipulate", "query_flags", "modifiers", "collation") data = dict((k, v) for k, v in iteritems(self.__dict__) if k.startswith('_Cursor__') and k[9:] in values_to_clone) if deepcopy: data = self._deepcopy(data) base.__dict__.update(data) return base def _clone_base(self, session): """Creates an empty Cursor object for information to be copied into. """ return self.__class__(self.__collection, session=session) def __die(self, synchronous=False): """Closes this cursor. """ already_killed = self.__killed self.__killed = True if self.__id and not already_killed: if self.__exhaust and self.__exhaust_mgr: # If this is an exhaust cursor and we haven't completely # exhausted the result set we *must* close the socket # to stop the server from sending more data. self.__exhaust_mgr.sock.close() else: address = _CursorAddress( self.__address, self.__collection.full_name) if synchronous: self.__collection.database.client._close_cursor_now( self.__id, address, session=self.__session) else: # The cursor will be closed later in a different session. self.__collection.database.client.close_cursor( self.__id, address) if self.__exhaust and self.__exhaust_mgr: self.__exhaust_mgr.close() if self.__session and not self.__explicit_session: self.__session._end_session(lock=synchronous) self.__session = None def close(self): """Explicitly close / kill this cursor. """ self.__die(True) def __query_spec(self): """Get the spec to use for a query. """ operators = self.__modifiers.copy() if self.__ordering: operators["$orderby"] = self.__ordering if self.__explain: operators["$explain"] = True if self.__hint: operators["$hint"] = self.__hint if self.__comment: operators["$comment"] = self.__comment if self.__max_scan: operators["$maxScan"] = self.__max_scan if self.__max_time_ms is not None: operators["$maxTimeMS"] = self.__max_time_ms if self.__max: operators["$max"] = self.__max if self.__min: operators["$min"] = self.__min if self.__return_key: operators["$returnKey"] = self.__return_key if self.__show_record_id: # This is upgraded to showRecordId for MongoDB 3.2+ "find" command. operators["$showDiskLoc"] = self.__show_record_id if self.__snapshot: operators["$snapshot"] = self.__snapshot if operators: # Make a shallow copy so we can cleanly rewind or clone. spec = self.__spec.copy() # White-listed commands must be wrapped in $query. if "$query" not in spec: # $query has to come first spec = SON([("$query", spec)]) if not isinstance(spec, SON): # Ensure the spec is SON. As order is important this will # ensure its set before merging in any extra operators. spec = SON(spec) spec.update(operators) return spec # Have to wrap with $query if "query" is the first key. # We can't just use $query anytime "query" is a key as # that breaks commands like count and find_and_modify. # Checking spec.keys()[0] covers the case that the spec # was passed as an instance of SON or OrderedDict. elif ("query" in self.__spec and (len(self.__spec) == 1 or next(iter(self.__spec)) == "query")): return SON({"$query": self.__spec}) return self.__spec def __check_okay_to_chain(self): """Check if it is okay to chain more options onto this cursor. """ if self.__retrieved or self.__id is not None: raise InvalidOperation("cannot set options after executing query") def add_option(self, mask): """Set arbitrary query flags using a bitmask. To set the tailable flag: cursor.add_option(2) """ if not isinstance(mask, int): raise TypeError("mask must be an int") self.__check_okay_to_chain() if mask & _QUERY_OPTIONS["exhaust"]: if self.__limit: raise InvalidOperation("Can't use limit and exhaust together.") if self.__collection.database.client.is_mongos: raise InvalidOperation('Exhaust cursors are ' 'not supported by mongos') self.__exhaust = True self.__query_flags |= mask return self def remove_option(self, mask): """Unset arbitrary query flags using a bitmask. To unset the tailable flag: cursor.remove_option(2) """ if not isinstance(mask, int): raise TypeError("mask must be an int") self.__check_okay_to_chain() if mask & _QUERY_OPTIONS["exhaust"]: self.__exhaust = False self.__query_flags &= ~mask return self def limit(self, limit): """Limits the number of results to be returned by this cursor. Raises :exc:`TypeError` if `limit` is not an integer. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. The last `limit` applied to this cursor takes precedence. A limit of ``0`` is equivalent to no limit. :Parameters: - `limit`: the number of results to return .. mongodoc:: limit """ if not isinstance(limit, integer_types): raise TypeError("limit must be an integer") if self.__exhaust: raise InvalidOperation("Can't use limit and exhaust together.") self.__check_okay_to_chain() self.__empty = False self.__limit = limit return self def batch_size(self, batch_size): """Limits the number of documents returned in one batch. Each batch requires a round trip to the server. It can be adjusted to optimize performance and limit data transfer. .. note:: batch_size can not override MongoDB's internal limits on the amount of data it will return to the client in a single batch (i.e if you set batch size to 1,000,000,000, MongoDB will currently only return 4-16MB of results per batch). Raises :exc:`TypeError` if `batch_size` is not an integer. Raises :exc:`ValueError` if `batch_size` is less than ``0``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. The last `batch_size` applied to this cursor takes precedence. :Parameters: - `batch_size`: The size of each batch of results requested. """ if not isinstance(batch_size, integer_types): raise TypeError("batch_size must be an integer") if batch_size < 0: raise ValueError("batch_size must be >= 0") self.__check_okay_to_chain() self.__batch_size = batch_size return self def skip(self, skip): """Skips the first `skip` results of this cursor. Raises :exc:`TypeError` if `skip` is not an integer. Raises :exc:`ValueError` if `skip` is less than ``0``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. The last `skip` applied to this cursor takes precedence. :Parameters: - `skip`: the number of results to skip """ if not isinstance(skip, integer_types): raise TypeError("skip must be an integer") if skip < 0: raise ValueError("skip must be >= 0") self.__check_okay_to_chain() self.__skip = skip return self def max_time_ms(self, max_time_ms): """Specifies a time limit for a query operation. If the specified time is exceeded, the operation will be aborted and :exc:`~pymongo.errors.ExecutionTimeout` is raised. If `max_time_ms` is ``None`` no limit is applied. Raises :exc:`TypeError` if `max_time_ms` is not an integer or ``None``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. :Parameters: - `max_time_ms`: the time limit after which the operation is aborted """ if (not isinstance(max_time_ms, integer_types) and max_time_ms is not None): raise TypeError("max_time_ms must be an integer or None") self.__check_okay_to_chain() self.__max_time_ms = max_time_ms return self def max_await_time_ms(self, max_await_time_ms): """Specifies a time limit for a getMore operation on a :attr:`~pymongo.cursor.CursorType.TAILABLE_AWAIT` cursor. For all other types of cursor max_await_time_ms is ignored. Raises :exc:`TypeError` if `max_await_time_ms` is not an integer or ``None``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. .. note:: `max_await_time_ms` requires server version **>= 3.2** :Parameters: - `max_await_time_ms`: the time limit after which the operation is aborted .. versionadded:: 3.2 """ if (not isinstance(max_await_time_ms, integer_types) and max_await_time_ms is not None): raise TypeError("max_await_time_ms must be an integer or None") self.__check_okay_to_chain() # Ignore max_await_time_ms if not tailable or await_data is False. if self.__query_flags & CursorType.TAILABLE_AWAIT: self.__max_await_time_ms = max_await_time_ms return self def __getitem__(self, index): """Get a single document or a slice of documents from this cursor. Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. To get a single document use an integral index, e.g.:: >>> db.test.find()[50] An :class:`IndexError` will be raised if the index is negative or greater than the amount of documents in this cursor. Any limit previously applied to this cursor will be ignored. To get a slice of documents use a slice index, e.g.:: >>> db.test.find()[20:25] This will return this cursor with a limit of ``5`` and skip of ``20`` applied. Using a slice index will override any prior limits or skips applied to this cursor (including those applied through previous calls to this method). Raises :class:`IndexError` when the slice has a step, a negative start value, or a stop value less than or equal to the start value. :Parameters: - `index`: An integer or slice index to be applied to this cursor """ self.__check_okay_to_chain() self.__empty = False if isinstance(index, slice): if index.step is not None: raise IndexError("Cursor instances do not support slice steps") skip = 0 if index.start is not None: if index.start < 0: raise IndexError("Cursor instances do not support " "negative indices") skip = index.start if index.stop is not None: limit = index.stop - skip if limit < 0: raise IndexError("stop index must be greater than start " "index for slice %r" % index) if limit == 0: self.__empty = True else: limit = 0 self.__skip = skip self.__limit = limit return self if isinstance(index, integer_types): if index < 0: raise IndexError("Cursor instances do not support negative " "indices") clone = self.clone() clone.skip(index + self.__skip) clone.limit(-1) # use a hard limit clone.__query_flags &= ~CursorType.TAILABLE_AWAIT # PYTHON-1371 for doc in clone: return doc raise IndexError("no such item for Cursor instance") raise TypeError("index %r cannot be applied to Cursor " "instances" % index) def max_scan(self, max_scan): """Limit the number of documents to scan when performing the query. Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. Only the last :meth:`max_scan` applied to this cursor has any effect. :Parameters: - `max_scan`: the maximum number of documents to scan """ self.__check_okay_to_chain() self.__max_scan = max_scan return self def max(self, spec): """Adds `max` operator that specifies upper bound for specific index. :Parameters: - `spec`: a list of field, limit pairs specifying the exclusive upper bound for all keys of a specific index in order. .. versionadded:: 2.7 """ if not isinstance(spec, (list, tuple)): raise TypeError("spec must be an instance of list or tuple") self.__check_okay_to_chain() self.__max = SON(spec) return self def min(self, spec): """Adds `min` operator that specifies lower bound for specific index. :Parameters: - `spec`: a list of field, limit pairs specifying the inclusive lower bound for all keys of a specific index in order. .. versionadded:: 2.7 """ if not isinstance(spec, (list, tuple)): raise TypeError("spec must be an instance of list or tuple") self.__check_okay_to_chain() self.__min = SON(spec) return self def sort(self, key_or_list, direction=None): """Sorts this cursor's results. Pass a field name and a direction, either :data:`~pymongo.ASCENDING` or :data:`~pymongo.DESCENDING`:: for doc in collection.find().sort('field', pymongo.ASCENDING): print(doc) To sort by multiple fields, pass a list of (key, direction) pairs:: for doc in collection.find().sort([ ('field1', pymongo.ASCENDING), ('field2', pymongo.DESCENDING)]): print(doc) Beginning with MongoDB version 2.6, text search results can be sorted by relevance:: cursor = db.test.find( {'$text': {'$search': 'some words'}}, {'score': {'$meta': 'textScore'}}) # Sort by 'score' field. cursor.sort([('score', {'$meta': 'textScore'})]) for doc in cursor: print(doc) Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. Only the last :meth:`sort` applied to this cursor has any effect. :Parameters: - `key_or_list`: a single key or a list of (key, direction) pairs specifying the keys to sort on - `direction` (optional): only used if `key_or_list` is a single key, if not given :data:`~pymongo.ASCENDING` is assumed """ self.__check_okay_to_chain() keys = helpers._index_list(key_or_list, direction) self.__ordering = helpers._index_document(keys) return self def count(self, with_limit_and_skip=False): """Get the size of the results set for this query. Returns the number of documents in the results set for this query. Does not take :meth:`limit` and :meth:`skip` into account by default - set `with_limit_and_skip` to ``True`` if that is the desired behavior. Raises :class:`~pymongo.errors.OperationFailure` on a database error. When used with MongoDB >= 2.6, :meth:`~count` uses any :meth:`~hint` applied to the query. In the following example the hint is passed to the count command: collection.find({'field': 'value'}).hint('field_1').count() The :meth:`count` method obeys the :attr:`~pymongo.collection.Collection.read_preference` of the :class:`~pymongo.collection.Collection` instance on which :meth:`~pymongo.collection.Collection.find` was called. :Parameters: - `with_limit_and_skip` (optional): take any :meth:`limit` or :meth:`skip` that has been applied to this cursor into account when getting the count .. note:: The `with_limit_and_skip` parameter requires server version **>= 1.1.4-** .. versionchanged:: 2.8 The :meth:`~count` method now supports :meth:`~hint`. """ validate_boolean("with_limit_and_skip", with_limit_and_skip) cmd = SON([("count", self.__collection.name), ("query", self.__spec)]) if self.__max_time_ms is not None: cmd["maxTimeMS"] = self.__max_time_ms if self.__comment: cmd["$comment"] = self.__comment if self.__hint is not None: cmd["hint"] = self.__hint if with_limit_and_skip: if self.__limit: cmd["limit"] = self.__limit if self.__skip: cmd["skip"] = self.__skip return self.__collection._count( cmd, self.__collation, session=self.__session) def distinct(self, key): """Get a list of distinct values for `key` among all documents in the result set of this query. Raises :class:`TypeError` if `key` is not an instance of :class:`basestring` (:class:`str` in python 3). The :meth:`distinct` method obeys the :attr:`~pymongo.collection.Collection.read_preference` of the :class:`~pymongo.collection.Collection` instance on which :meth:`~pymongo.collection.Collection.find` was called. :Parameters: - `key`: name of key for which we want to get the distinct values .. seealso:: :meth:`pymongo.collection.Collection.distinct` """ options = {} if self.__spec: options["query"] = self.__spec if self.__max_time_ms is not None: options['maxTimeMS'] = self.__max_time_ms if self.__comment: options['$comment'] = self.__comment if self.__collation is not None: options['collation'] = self.__collation return self.__collection.distinct( key, session=self.__session, **options) def explain(self): """Returns an explain plan record for this cursor. .. mongodoc:: explain """ c = self.clone() c.__explain = True # always use a hard limit for explains if c.__limit: c.__limit = -abs(c.__limit) return next(c) def __set_hint(self, index): if index is None: self.__hint = None return if isinstance(index, string_type): self.__hint = index else: self.__hint = helpers._index_document(index) def hint(self, index): """Adds a 'hint', telling Mongo the proper index to use for the query. Judicious use of hints can greatly improve query performance. When doing a query on multiple fields (at least one of which is indexed) pass the indexed field as a hint to the query. Hinting will not do anything if the corresponding index does not exist. Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. `index` should be an index as passed to :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``) or the name of the index. If `index` is ``None`` any existing hint for this query is cleared. The last hint applied to this cursor takes precedence over all others. :Parameters: - `index`: index to hint on (as an index specifier) .. versionchanged:: 2.8 The :meth:`~hint` method accepts the name of the index. """ self.__check_okay_to_chain() self.__set_hint(index) return self def comment(self, comment): """Adds a 'comment' to the cursor. http://docs.mongodb.org/manual/reference/operator/comment/ :Parameters: - `comment`: A string or document .. versionadded:: 2.7 """ self.__check_okay_to_chain() self.__comment = comment return self def where(self, code): """Adds a $where clause to this query. The `code` argument must be an instance of :class:`basestring` (:class:`str` in python 3) or :class:`~bson.code.Code` containing a JavaScript expression. This expression will be evaluated for each document scanned. Only those documents for which the expression evaluates to *true* will be returned as results. The keyword *this* refers to the object currently being scanned. Raises :class:`TypeError` if `code` is not an instance of :class:`basestring` (:class:`str` in python 3). Raises :class:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. Only the last call to :meth:`where` applied to a :class:`Cursor` has any effect. :Parameters: - `code`: JavaScript expression to use as a filter """ self.__check_okay_to_chain() if not isinstance(code, Code): code = Code(code) self.__spec["$where"] = code return self def collation(self, collation): """Adds a :class:`~pymongo.collation.Collation` to this query. This option is only supported on MongoDB 3.4 and above. Raises :exc:`TypeError` if `collation` is not an instance of :class:`~pymongo.collation.Collation` or a ``dict``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. Only the last collation applied to this cursor has any effect. :Parameters: - `collation`: An instance of :class:`~pymongo.collation.Collation`. """ self.__check_okay_to_chain() self.__collation = validate_collation_or_none(collation) return self def __send_message(self, operation): """Send a query or getmore operation and handles the response. If operation is ``None`` this is an exhaust cursor, which reads the next result batch off the exhaust socket instead of sending getMore messages to the server. Can raise ConnectionFailure. """ client = self.__collection.database.client listeners = client._event_listeners publish = listeners.enabled_for_commands from_command = False start = datetime.datetime.now() def duration(): return datetime.datetime.now() - start if operation: kwargs = { "read_preference": self.__read_preference, "exhaust": self.__exhaust, } if self.__address is not None: kwargs["address"] = self.__address try: response = client._send_message_with_response(operation, **kwargs) self.__address = response.address if self.__exhaust: # 'response' is an ExhaustResponse. self.__exhaust_mgr = _SocketManager(response.socket_info, response.pool) cmd_name = operation.name reply = response.data rqst_id = response.request_id from_command = response.from_command except AutoReconnect: # Don't try to send kill cursors on another socket # or to another server. It can cause a _pinValue # assertion on some server releases if we get here # due to a socket timeout. self.__killed = True self.__die() raise else: # Exhaust cursor - no getMore message. rqst_id = 0 cmd_name = 'getMore' if publish: # Fake a getMore command. cmd = SON([('getMore', self.__id), ('collection', self.__collection.name)]) if self.__batch_size: cmd['batchSize'] = self.__batch_size if self.__max_time_ms: cmd['maxTimeMS'] = self.__max_time_ms listeners.publish_command_start( cmd, self.__collection.database.name, 0, self.__address) try: reply = self.__exhaust_mgr.sock.receive_message(None) except Exception as exc: if publish: listeners.publish_command_failure( duration(), _convert_exception(exc), cmd_name, rqst_id, self.__address) if isinstance(exc, ConnectionFailure): self.__die() raise try: docs = self._unpack_response(response=reply, cursor_id=self.__id, codec_options=self.__codec_options) if from_command: first = docs[0] client._receive_cluster_time(first, self.__session) helpers._check_command_response(first) except OperationFailure as exc: self.__killed = True # Make sure exhaust socket is returned immediately, if necessary. self.__die() if publish: listeners.publish_command_failure( duration(), exc.details, cmd_name, rqst_id, self.__address) # If this is a tailable cursor the error is likely # due to capped collection roll over. Setting # self.__killed to True ensures Cursor.alive will be # False. No need to re-raise. if self.__query_flags & _QUERY_OPTIONS["tailable_cursor"]: return raise except NotMasterError as exc: # Don't send kill cursors to another server after a "not master" # error. It's completely pointless. self.__killed = True # Make sure exhaust socket is returned immediately, if necessary. self.__die() if publish: listeners.publish_command_failure( duration(), exc.details, cmd_name, rqst_id, self.__address) client._reset_server_and_request_check(self.__address) raise except Exception as exc: if publish: listeners.publish_command_failure( duration(), _convert_exception(exc), cmd_name, rqst_id, self.__address) raise if publish: # Must publish in find / getMore / explain command response format. if from_command: res = docs[0] elif cmd_name == "explain": res = docs[0] if reply.number_returned else {} else: res = {"cursor": {"id": reply.cursor_id, "ns": self.__collection.full_name}, "ok": 1} if cmd_name == "find": res["cursor"]["firstBatch"] = docs else: res["cursor"]["nextBatch"] = docs listeners.publish_command_success( duration(), res, cmd_name, rqst_id, self.__address) if from_command and cmd_name != "explain": cursor = docs[0]['cursor'] self.__id = cursor['id'] if cmd_name == 'find': documents = cursor['firstBatch'] else: documents = cursor['nextBatch'] self.__data = deque(documents) self.__retrieved += len(documents) else: self.__id = reply.cursor_id self.__data = deque(docs) self.__retrieved += reply.number_returned if self.__id == 0: self.__killed = True # Don't wait for garbage collection to call __del__, return the # socket and the session to the pool now. self.__die() if self.__limit and self.__id and self.__limit <= self.__retrieved: self.__die() def _unpack_response(self, response, cursor_id, codec_options): return response.unpack_response(cursor_id, codec_options) def _refresh(self): """Refreshes the cursor with more data from Mongo. Returns the length of self.__data after refresh. Will exit early if self.__data is already non-empty. Raises OperationFailure when the cursor cannot be refreshed due to an error on the query. """ if len(self.__data) or self.__killed: return len(self.__data) if not self.__session: self.__session = self.__collection.database.client._ensure_session() if self.__id is None: # Query q = self._query_class(self.__query_flags, self.__collection.database.name, self.__collection.name, self.__skip, self.__query_spec(), self.__projection, self.__codec_options, self.__read_preference, self.__limit, self.__batch_size, self.__read_concern, self.__collation, self.__session, self.__collection.database.client) self.__send_message(q) elif self.__id: # Get More if self.__limit: limit = self.__limit - self.__retrieved if self.__batch_size: limit = min(limit, self.__batch_size) else: limit = self.__batch_size # Exhaust cursors don't send getMore messages. if self.__exhaust: self.__send_message(None) else: g = self._getmore_class(self.__collection.database.name, self.__collection.name, limit, self.__id, self.__codec_options, self.__session, self.__collection.database.client, self.__max_await_time_ms) self.__send_message(g) return len(self.__data) @property def alive(self): """Does this cursor have the potential to return more data? This is mostly useful with `tailable cursors `_ since they will stop iterating even though they *may* return more results in the future. With regular cursors, simply use a for loop instead of :attr:`alive`:: for doc in collection.find(): print(doc) .. note:: Even if :attr:`alive` is True, :meth:`next` can raise :exc:`StopIteration`. :attr:`alive` can also be True while iterating a cursor from a failed server. In this case :attr:`alive` will return False after :meth:`next` fails to retrieve the next batch of results from the server. """ return bool(len(self.__data) or (not self.__killed)) @property def cursor_id(self): """Returns the id of the cursor Useful if you need to manage cursor ids and want to handle killing cursors manually using :meth:`~pymongo.mongo_client.MongoClient.kill_cursors` .. versionadded:: 2.2 """ return self.__id @property def address(self): """The (host, port) of the server used, or None. .. versionchanged:: 3.0 Renamed from "conn_id". """ return self.__address @property def session(self): """The cursor's :class:`~pymongo.client_session.ClientSession`, or None. .. versionadded:: 3.6 """ if self.__explicit_session: return self.__session def __iter__(self): return self def next(self): """Advance the cursor.""" if self.__empty: raise StopIteration if len(self.__data) or self._refresh(): if self.__manipulate: _db = self.__collection.database return _db._fix_outgoing(self.__data.popleft(), self.__collection) else: return self.__data.popleft() else: raise StopIteration __next__ = next def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def __copy__(self): """Support function for `copy.copy()`. .. versionadded:: 2.4 """ return self._clone(deepcopy=False) def __deepcopy__(self, memo): """Support function for `copy.deepcopy()`. .. versionadded:: 2.4 """ return self._clone(deepcopy=True) def _deepcopy(self, x, memo=None): """Deepcopy helper for the data dictionary or list. Regular expressions cannot be deep copied but as they are immutable we don't have to copy them when cloning. """ if not hasattr(x, 'items'): y, is_list, iterator = [], True, enumerate(x) else: y, is_list, iterator = {}, False, iteritems(x) if memo is None: memo = {} val_id = id(x) if val_id in memo: return memo.get(val_id) memo[val_id] = y for key, value in iterator: if isinstance(value, (dict, list)) and not isinstance(value, SON): value = self._deepcopy(value, memo) elif not isinstance(value, RE_TYPE): value = copy.deepcopy(value, memo) if is_list: y.append(value) else: if not isinstance(key, RE_TYPE): key = copy.deepcopy(key, memo) y[key] = value return y class RawBatchCursor(Cursor): """A cursor / iterator over raw batches of BSON data from a query result.""" _query_class = _RawBatchQuery _getmore_class = _RawBatchGetMore def __init__(self, *args, **kwargs): """Create a new cursor / iterator over raw batches of BSON data. Should not be called directly by application developers - see :meth:`~pymongo.collection.Collection.find_raw_batches` instead. .. mongodoc:: cursors """ manipulate = kwargs.get('manipulate') kwargs['manipulate'] = False super(RawBatchCursor, self).__init__(*args, **kwargs) # Throw only after cursor's initialized, to prevent errors in __del__. if manipulate: raise InvalidOperation( "Cannot use RawBatchCursor with manipulate=True") def _unpack_response(self, response, cursor_id, codec_options): return response.raw_response(cursor_id) def explain(self): """Returns an explain plan record for this cursor. .. mongodoc:: explain """ clone = self._clone(deepcopy=True, base=Cursor(self.collection)) return clone.explain() def __getitem__(self, index): raise InvalidOperation("Cannot call __getitem__ on RawBatchCursor") pymongo-3.6.1/pymongo/read_concern.py0000644000076600000240000000444113245617773020141 0ustar shanestaff00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License", # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for working with read concerns.""" from bson.py3compat import string_type class ReadConcern(object): """ReadConcern :Parameters: - `level`: (string) The read concern level specifies the level of isolation for read operations. For example, a read operation using a read concern level of ``majority`` will only return data that has been written to a majority of nodes. If the level is left unspecified, the server default will be used. .. versionadded:: 3.2 """ def __init__(self, level=None): if level is None or isinstance(level, string_type): self.__level = level else: raise TypeError( 'level must be a string or None.') @property def level(self): """The read concern level.""" return self.__level @property def ok_for_legacy(self): """Return ``True`` if this read concern is compatible with old wire protocol versions.""" return self.level is None or self.level == 'local' @property def document(self): """The document representation of this read concern. .. note:: :class:`ReadConcern` is immutable. Mutating the value of :attr:`document` does not mutate this :class:`ReadConcern`. """ doc = {} if self.__level: doc['level'] = self.level return doc def __eq__(self, other): if isinstance(other, ReadConcern): return self.document == other.document return NotImplemented def __repr__(self): if self.level: return 'ReadConcern(%s)' % self.level return 'ReadConcern()' DEFAULT_READ_CONCERN = ReadConcern() pymongo-3.6.1/pymongo/server_selectors.py0000644000076600000240000001227313245617773021112 0ustar shanestaff00000000000000# Copyright 2014-2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Criteria to select some ServerDescriptions from a TopologyDescription.""" from pymongo.server_type import SERVER_TYPE class Selection(object): """Input or output of a server selector function.""" @classmethod def from_topology_description(cls, topology_description): known_servers = topology_description.known_servers primary = None for sd in known_servers: if sd.server_type == SERVER_TYPE.RSPrimary: primary = sd break return Selection(topology_description, topology_description.known_servers, topology_description.common_wire_version, primary) def __init__(self, topology_description, server_descriptions, common_wire_version, primary): self.topology_description = topology_description self.server_descriptions = server_descriptions self.primary = primary self.common_wire_version = common_wire_version def with_server_descriptions(self, server_descriptions): return Selection(self.topology_description, server_descriptions, self.common_wire_version, self.primary) def secondary_with_max_last_write_date(self): secondaries = secondary_server_selector(self) if secondaries.server_descriptions: return max(secondaries.server_descriptions, key=lambda sd: sd.last_write_date) @property def primary_selection(self): primaries = [self.primary] if self.primary else [] return self.with_server_descriptions(primaries) @property def heartbeat_frequency(self): return self.topology_description.heartbeat_frequency @property def topology_type(self): return self.topology_description.topology_type def __bool__(self): return bool(self.server_descriptions) __nonzero__ = __bool__ # Python 2. def __getitem__(self, item): return self.server_descriptions[item] def any_server_selector(selection): return selection def readable_server_selector(selection): return selection.with_server_descriptions( [s for s in selection.server_descriptions if s.is_readable]) def writable_server_selector(selection): return selection.with_server_descriptions( [s for s in selection.server_descriptions if s.is_writable]) def secondary_server_selector(selection): return selection.with_server_descriptions( [s for s in selection.server_descriptions if s.server_type == SERVER_TYPE.RSSecondary]) def arbiter_server_selector(selection): return selection.with_server_descriptions( [s for s in selection.server_descriptions if s.server_type == SERVER_TYPE.RSArbiter]) def writable_preferred_server_selector(selection): """Like PrimaryPreferred but doesn't use tags or latency.""" return (writable_server_selector(selection) or secondary_server_selector(selection)) def apply_single_tag_set(tag_set, selection): """All servers matching one tag set. A tag set is a dict. A server matches if its tags are a superset: A server tagged {'a': '1', 'b': '2'} matches the tag set {'a': '1'}. The empty tag set {} matches any server. """ def tags_match(server_tags): for key, value in tag_set.items(): if key not in server_tags or server_tags[key] != value: return False return True return selection.with_server_descriptions( [s for s in selection.server_descriptions if tags_match(s.tags)]) def apply_tag_sets(tag_sets, selection): """All servers match a list of tag sets. tag_sets is a list of dicts. The empty tag set {} matches any server, and may be provided at the end of the list as a fallback. So [{'a': 'value'}, {}] expresses a preference for servers tagged {'a': 'value'}, but accepts any server if none matches the first preference. """ for tag_set in tag_sets: with_tag_set = apply_single_tag_set(tag_set, selection) if with_tag_set: return with_tag_set return selection.with_server_descriptions([]) def secondary_with_tags_server_selector(tag_sets, selection): """All near-enough secondaries matching the tag sets.""" return apply_tag_sets(tag_sets, secondary_server_selector(selection)) def member_with_tags_server_selector(tag_sets, selection): """All near-enough members matching the tag sets.""" return apply_tag_sets(tag_sets, readable_server_selector(selection)) pymongo-3.6.1/THIRD-PARTY-NOTICES0000644000076600000240000001515613245621354016244 0ustar shanestaff00000000000000PyMongo uses third-party libraries or other resources that may be distributed under licenses different than the PyMongo software. In the event that we accidentally failed to list a required notice, please bring it to our attention through any of the ways detailed here: mongodb-dev@googlegroups.com The attached notices are provided for information only. For any licenses that require disclosure of source, sources are available at https://github.com/mongodb/mongo-python-driver. 1) License Notice for time64.c ------------------------------ Copyright (c) 2007-2010 Michael G Schwern This software originally derived from Paul Sheer's pivotal_gmtime_r.c. The MIT License: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 2) License Notice for bson-stdint-win32.h ----------------------------------------- ISO C9x compliant stdint.h for Microsoft Visual Studio Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 Copyright (c) 2006-2013 Alexander Chemeris Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the product nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3) License Notice for encoding_helpers.c ---------------------------------------- Portions Copyright 2001 Unicode, Inc. Disclaimer This source code is provided as is by Unicode, Inc. No claims are made as to fitness for any particular purpose. No warranties of any kind are expressed or implied. The recipient agrees to determine applicability of information provided. If this file has been purchased on magnetic or optical media from Unicode, Inc., the sole remedy for any claim will be exchange of defective media within 90 days of receipt. Limitations on Rights to Redistribute This Code Unicode, Inc. hereby grants the right to freely use the information supplied in this file in the creation of products supporting the Unicode Standard, and to make copies of this file in any form for internal or external distribution as long as this notice remains attached. 4) License Notice for ssl_match_hostname.py ------------------------------------------- Python License (Python-2.0) Python License, Version 2 (Python-2.0) PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 -------------------------------------------- 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001-2013 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python. 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. pymongo-3.6.1/setup.py0000755000076600000240000003563513246100114015154 0ustar shanestaff00000000000000import os import platform import re import sys import warnings # Hack to silence atexit traceback in some Python versions try: import multiprocessing except ImportError: pass # Don't force people to install setuptools unless # we have to. try: from setuptools import setup except ImportError: from ez_setup import use_setuptools use_setuptools() from setuptools import setup from distutils.cmd import Command from distutils.command.build_ext import build_ext from distutils.errors import CCompilerError, DistutilsOptionError from distutils.errors import DistutilsPlatformError, DistutilsExecError from distutils.core import Extension try: import sphinx _HAVE_SPHINX = True except ImportError: _HAVE_SPHINX = False version = "3.6.1" f = open("README.rst") try: try: readme_content = f.read() except: readme_content = "" finally: f.close() # PYTHON-654 - Clang doesn't support -mno-fused-madd but the pythons Apple # ships are built with it. This is a problem starting with Xcode 5.1 # since clang 3.4 errors out when it encounters unrecognized compiler # flags. This hack removes -mno-fused-madd from the CFLAGS automatically # generated by distutils for Apple provided pythons, allowing C extension # builds to complete without error. The inspiration comes from older # versions of distutils.sysconfig.get_config_vars. if sys.platform == 'darwin' and 'clang' in platform.python_compiler().lower(): from distutils.sysconfig import get_config_vars res = get_config_vars() for key in ('CFLAGS', 'PY_CFLAGS'): if key in res: flags = res[key] flags = re.sub('-mno-fused-madd', '', flags) res[key] = flags class test(Command): description = "run the tests" user_options = [ ("test-module=", "m", "Discover tests in specified module"), ("test-suite=", "s", "Test suite to run (e.g. 'some_module.test_suite')"), ("failfast", "f", "Stop running tests on first failure or error"), ("xunit-output=", "x", "Generate a results directory with XUnit XML format") ] def initialize_options(self): self.test_module = None self.test_suite = None self.failfast = False self.xunit_output = None def finalize_options(self): if self.test_suite is None and self.test_module is None: self.test_module = 'test' elif self.test_module is not None and self.test_suite is not None: raise DistutilsOptionError( "You may specify a module or suite, but not both" ) def run(self): # Installing required packages, running egg_info and build_ext are # part of normal operation for setuptools.command.test.test if self.distribution.install_requires: self.distribution.fetch_build_eggs( self.distribution.install_requires) if self.distribution.tests_require: self.distribution.fetch_build_eggs(self.distribution.tests_require) if self.xunit_output: if sys.version_info[:2] == (2, 6): self.distribution.fetch_build_eggs( ["unittest-xml-reporting>=1.14.0,<2.0.0a0"]) else: self.distribution.fetch_build_eggs(["unittest-xml-reporting"]) self.run_command('egg_info') build_ext_cmd = self.reinitialize_command('build_ext') build_ext_cmd.inplace = 1 self.run_command('build_ext') # Construct a TextTestRunner directly from the unittest imported from # test (this will be unittest2 under Python 2.6), which creates a # TestResult that supports the 'addSkip' method. setuptools will by # default create a TextTestRunner that uses the old TestResult class, # resulting in DeprecationWarnings instead of skipping tests under 2.6. from test import unittest, PymongoTestRunner, test_cases if self.test_suite is None: all_tests = unittest.defaultTestLoader.discover(self.test_module) suite = unittest.TestSuite() suite.addTests(sorted(test_cases(all_tests), key=lambda x: x.__module__)) else: suite = unittest.defaultTestLoader.loadTestsFromName( self.test_suite) if self.xunit_output: from xmlrunner import XMLTestRunner runner = XMLTestRunner(verbosity=2, failfast=self.failfast, output=self.xunit_output) else: runner = PymongoTestRunner(verbosity=2, failfast=self.failfast) result = runner.run(suite) sys.exit(not result.wasSuccessful()) class doc(Command): description = "generate or test documentation" user_options = [("test", "t", "run doctests instead of generating documentation")] boolean_options = ["test"] def initialize_options(self): self.test = False def finalize_options(self): pass def run(self): if not _HAVE_SPHINX: raise RuntimeError( "You must install Sphinx to build or test the documentation.") if sys.version_info[0] >= 3: import doctest from doctest import OutputChecker as _OutputChecker # Match u or U (possibly followed by r or R), removing it. # r/R can follow u/U but not precede it. Don't match the # single character string 'u' or 'U'. _u_literal_re = re.compile( r"(\W|^)(? (2, 6): # 2.6's distutils.msvc9compiler can raise an IOError when failing to # find the compiler build_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError) else: build_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError) class custom_build_ext(build_ext): """Allow C extension building to fail. The C extension speeds up BSON encoding, but is not essential. """ warning_message = """ ******************************************************************** WARNING: %s could not be compiled. No C extensions are essential for PyMongo to run, although they do result in significant speed improvements. %s Please see the installation docs for solutions to build issues: http://api.mongodb.org/python/current/installation.html Here are some hints for popular operating systems: If you are seeing this message on Linux you probably need to install GCC and/or the Python development package for your version of Python. Debian and Ubuntu users should issue the following command: $ sudo apt-get install build-essential python-dev Users of Red Hat based distributions (RHEL, CentOS, Amazon Linux, Oracle Linux, Fedora, etc.) should issue the following command: $ sudo yum install gcc python-devel If you are seeing this message on Microsoft Windows please install PyMongo using pip. Modern versions of pip will install PyMongo from binary wheels available on pypi. If you must install from source read the documentation here: https://api.mongodb.com/python/current/installation.html#installing-from-source-on-windows If you are seeing this message on macOS / OSX please install PyMongo using pip. Modern versions of pip will install PyMongo from binary wheels available on pypi. If wheels are not available for your version of macOS / OSX, or you must install from source read the documentation here: http://api.mongodb.org/python/current/installation.html#osx ******************************************************************** """ def run(self): try: build_ext.run(self) except DistutilsPlatformError: e = sys.exc_info()[1] sys.stdout.write('%s\n' % str(e)) warnings.warn(self.warning_message % ("Extension modules", "There was an issue with " "your platform configuration" " - see above.")) def build_extension(self, ext): name = ext.name if sys.version_info[:3] >= (2, 6, 0): try: build_ext.build_extension(self, ext) except build_errors: e = sys.exc_info()[1] sys.stdout.write('%s\n' % str(e)) warnings.warn(self.warning_message % ("The %s extension " "module" % (name,), "The output above " "this warning shows how " "the compilation " "failed.")) else: warnings.warn(self.warning_message % ("The %s extension " "module" % (name,), "PyMongo supports python " ">= 2.6.")) ext_modules = [Extension('bson._cbson', include_dirs=['bson'], sources=['bson/_cbsonmodule.c', 'bson/time64.c', 'bson/buffer.c', 'bson/encoding_helpers.c']), Extension('pymongo._cmessage', include_dirs=['bson'], sources=['pymongo/_cmessagemodule.c', 'bson/buffer.c'])] vi = sys.version_info if vi[0] == 2: extras_require = {'tls': ["ipaddress"], 'srv': ["dnspython>=1.8.0,<2.0.0"]} else: extras_require = {'tls': [], 'srv': ["dnspython>=1.13.0,<2.0.0"]} if sys.platform == 'win32': extras_require['gssapi'] = ["winkerberos>=0.5.0"] if vi[0] == 2 and vi < (2, 7, 9) or vi[0] == 3 and vi < (3, 4): extras_require['tls'].append("wincertstore>=0.2") else: extras_require['gssapi'] = ["pykerberos"] if vi[0] == 2 and vi < (2, 7, 9): extras_require['tls'].append("certifi") extra_opts = { "packages": ["bson", "pymongo", "gridfs"] } if sys.version_info[:2] == (2, 6): try: import unittest2 except ImportError: # The setuptools version on Solaris 11 is incapable # of recognizing if unittest2 is already installed. # It's also incapable of installing any version of # unittest2 newer than 0.8.0 extra_opts['tests_require'] = "unittest2<=0.8.0" if "--no_ext" in sys.argv: sys.argv.remove("--no_ext") elif (sys.platform.startswith("java") or sys.platform == "cli" or "PyPy" in sys.version): sys.stdout.write(""" *****************************************************\n The optional C extensions are currently not supported\n by this python implementation.\n *****************************************************\n """) else: extra_opts['ext_modules'] = ext_modules setup( name="pymongo", version=version, description="Python driver for MongoDB ", long_description=readme_content, author="Mike Dirolf", author_email="mongodb-user@googlegroups.com", maintainer="Bernie Hackett", maintainer_email="bernie@mongodb.com", url="http://github.com/mongodb/mongo-python-driver", keywords=["mongo", "mongodb", "pymongo", "gridfs", "bson"], install_requires=[], license="Apache License, Version 2.0", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Database"], cmdclass={"build_ext": custom_build_ext, "doc": doc, "test": test}, extras_require=extras_require, **extra_opts ) pymongo-3.6.1/gridfs/0000755000076600000240000000000013246104133014706 5ustar shanestaff00000000000000pymongo-3.6.1/gridfs/grid_file.py0000644000076600000240000006247313245621354017230 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing files stored in GridFS.""" import datetime import math import os from hashlib import md5 from bson.son import SON from bson.binary import Binary from bson.objectid import ObjectId from bson.py3compat import text_type, StringIO from gridfs.errors import CorruptGridFile, FileExists, NoFile from pymongo import ASCENDING from pymongo.collection import Collection from pymongo.cursor import Cursor from pymongo.errors import (ConfigurationError, DuplicateKeyError, OperationFailure) from pymongo.read_preferences import ReadPreference try: _SEEK_SET = os.SEEK_SET _SEEK_CUR = os.SEEK_CUR _SEEK_END = os.SEEK_END # before 2.5 except AttributeError: _SEEK_SET = 0 _SEEK_CUR = 1 _SEEK_END = 2 EMPTY = b"" NEWLN = b"\n" """Default chunk size, in bytes.""" # Slightly under a power of 2, to work well with server's record allocations. DEFAULT_CHUNK_SIZE = 255 * 1024 _C_INDEX = SON([("files_id", ASCENDING), ("n", ASCENDING)]) _F_INDEX = SON([("filename", ASCENDING), ("uploadDate", ASCENDING)]) def _grid_in_property(field_name, docstring, read_only=False, closed_only=False): """Create a GridIn property.""" def getter(self): if closed_only and not self._closed: raise AttributeError("can only get %r on a closed file" % field_name) # Protect against PHP-237 if field_name == 'length': return self._file.get(field_name, 0) return self._file.get(field_name, None) def setter(self, value): if self._closed: self._coll.files.update_one({"_id": self._file["_id"]}, {"$set": {field_name: value}}) self._file[field_name] = value if read_only: docstring += "\n\nThis attribute is read-only." elif closed_only: docstring = "%s\n\n%s" % (docstring, "This attribute is read-only and " "can only be read after :meth:`close` " "has been called.") if not read_only and not closed_only: return property(getter, setter, doc=docstring) return property(getter, doc=docstring) def _grid_out_property(field_name, docstring): """Create a GridOut property.""" def getter(self): self._ensure_file() # Protect against PHP-237 if field_name == 'length': return self._file.get(field_name, 0) return self._file.get(field_name, None) docstring += "\n\nThis attribute is read-only." return property(getter, doc=docstring) class GridIn(object): """Class to write data to GridFS. """ def __init__(self, root_collection, session=None, **kwargs): """Write a file to GridFS Application developers should generally not need to instantiate this class directly - instead see the methods provided by :class:`~gridfs.GridFS`. Raises :class:`TypeError` if `root_collection` is not an instance of :class:`~pymongo.collection.Collection`. Any of the file level options specified in the `GridFS Spec `_ may be passed as keyword arguments. Any additional keyword arguments will be set as additional fields on the file document. Valid keyword arguments include: - ``"_id"``: unique ID for this file (default: :class:`~bson.objectid.ObjectId`) - this ``"_id"`` must not have already been used for another file - ``"filename"``: human name for the file - ``"contentType"`` or ``"content_type"``: valid mime-type for the file - ``"chunkSize"`` or ``"chunk_size"``: size of each of the chunks, in bytes (default: 255 kb) - ``"encoding"``: encoding used for this file. In Python 2, any :class:`unicode` that is written to the file will be converted to a :class:`str`. In Python 3, any :class:`str` that is written to the file will be converted to :class:`bytes`. :Parameters: - `root_collection`: root collection to write to - `session` (optional): a :class:`~pymongo.client_session.ClientSession` to use for all commands - `**kwargs` (optional): file level options (see above) .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.0 `root_collection` must use an acknowledged :attr:`~pymongo.collection.Collection.write_concern` """ if not isinstance(root_collection, Collection): raise TypeError("root_collection must be an " "instance of Collection") # With w=0, 'filemd5' might run before the final chunks are written. if not root_collection.write_concern.acknowledged: raise ConfigurationError('root_collection must use ' 'acknowledged write_concern') # Handle alternative naming if "content_type" in kwargs: kwargs["contentType"] = kwargs.pop("content_type") if "chunk_size" in kwargs: kwargs["chunkSize"] = kwargs.pop("chunk_size") coll = root_collection.with_options( read_preference=ReadPreference.PRIMARY) kwargs['md5'] = md5() # Defaults kwargs["_id"] = kwargs.get("_id", ObjectId()) kwargs["chunkSize"] = kwargs.get("chunkSize", DEFAULT_CHUNK_SIZE) object.__setattr__(self, "_session", session) object.__setattr__(self, "_coll", coll) object.__setattr__(self, "_chunks", coll.chunks) object.__setattr__(self, "_file", kwargs) object.__setattr__(self, "_buffer", StringIO()) object.__setattr__(self, "_position", 0) object.__setattr__(self, "_chunk_number", 0) object.__setattr__(self, "_closed", False) object.__setattr__(self, "_ensured_index", False) def __create_index(self, collection, index_key, unique): doc = collection.find_one(projection={"_id": 1}, session=self._session) if doc is None: try: index_keys = [index_spec['key'] for index_spec in collection.list_indexes(session=self._session)] except OperationFailure: index_keys = [] if index_key not in index_keys: collection.create_index( index_key.items(), unique=unique, session=self._session) def __ensure_indexes(self): if not object.__getattribute__(self, "_ensured_index"): self.__create_index(self._coll.files, _F_INDEX, False) self.__create_index(self._coll.chunks, _C_INDEX, True) object.__setattr__(self, "_ensured_index", True) def abort(self): """Remove all chunks/files that may have been uploaded and close. """ self._coll.chunks.delete_many( {"files_id": self._file['_id']}, session=self._session) self._coll.files.delete_one( {"_id": self._file['_id']}, session=self._session) object.__setattr__(self, "_closed", True) @property def closed(self): """Is this file closed? """ return self._closed _id = _grid_in_property("_id", "The ``'_id'`` value for this file.", read_only=True) filename = _grid_in_property("filename", "Name of this file.") name = _grid_in_property("filename", "Alias for `filename`.") content_type = _grid_in_property("contentType", "Mime-type for this file.") length = _grid_in_property("length", "Length (in bytes) of this file.", closed_only=True) chunk_size = _grid_in_property("chunkSize", "Chunk size for this file.", read_only=True) upload_date = _grid_in_property("uploadDate", "Date that this file was uploaded.", closed_only=True) md5 = _grid_in_property("md5", "MD5 of the contents of this file " "(generated on the server).", closed_only=True) def __getattr__(self, name): if name in self._file: return self._file[name] raise AttributeError("GridIn object has no attribute '%s'" % name) def __setattr__(self, name, value): # For properties of this instance like _buffer, or descriptors set on # the class like filename, use regular __setattr__ if name in self.__dict__ or name in self.__class__.__dict__: object.__setattr__(self, name, value) else: # All other attributes are part of the document in db.fs.files. # Store them to be sent to server on close() or if closed, send # them now. self._file[name] = value if self._closed: self._coll.files.update_one({"_id": self._file["_id"]}, {"$set": {name: value}}) def __flush_data(self, data): """Flush `data` to a chunk. """ # Ensure the index, even if there's nothing to write, so # the filemd5 command always succeeds. self.__ensure_indexes() self._file['md5'].update(data) if not data: return assert(len(data) <= self.chunk_size) chunk = {"files_id": self._file["_id"], "n": self._chunk_number, "data": Binary(data)} try: self._chunks.insert_one(chunk, session=self._session) except DuplicateKeyError: self._raise_file_exists(self._file['_id']) self._chunk_number += 1 self._position += len(data) def __flush_buffer(self): """Flush the buffer contents out to a chunk. """ self.__flush_data(self._buffer.getvalue()) self._buffer.close() self._buffer = StringIO() def __flush(self): """Flush the file to the database. """ try: self.__flush_buffer() self._file['md5'] = self._file["md5"].hexdigest() self._file["length"] = self._position self._file["uploadDate"] = datetime.datetime.utcnow() return self._coll.files.insert_one( self._file, session=self._session) except DuplicateKeyError: self._raise_file_exists(self._id) def _raise_file_exists(self, file_id): """Raise a FileExists exception for the given file_id.""" raise FileExists("file with _id %r already exists" % file_id) def close(self): """Flush the file and close it. A closed file cannot be written any more. Calling :meth:`close` more than once is allowed. """ if not self._closed: self.__flush() object.__setattr__(self, "_closed", True) def write(self, data): """Write data to the file. There is no return value. `data` can be either a string of bytes or a file-like object (implementing :meth:`read`). If the file has an :attr:`encoding` attribute, `data` can also be a :class:`unicode` (:class:`str` in python 3) instance, which will be encoded as :attr:`encoding` before being written. Due to buffering, the data may not actually be written to the database until the :meth:`close` method is called. Raises :class:`ValueError` if this file is already closed. Raises :class:`TypeError` if `data` is not an instance of :class:`str` (:class:`bytes` in python 3), a file-like object, or an instance of :class:`unicode` (:class:`str` in python 3). Unicode data is only allowed if the file has an :attr:`encoding` attribute. :Parameters: - `data`: string of bytes or file-like object to be written to the file """ if self._closed: raise ValueError("cannot write to a closed file") try: # file-like read = data.read except AttributeError: # string if not isinstance(data, (text_type, bytes)): raise TypeError("can only write strings or file-like objects") if isinstance(data, text_type): try: data = data.encode(self.encoding) except AttributeError: raise TypeError("must specify an encoding for file in " "order to write %s" % (text_type.__name__,)) read = StringIO(data).read if self._buffer.tell() > 0: # Make sure to flush only when _buffer is complete space = self.chunk_size - self._buffer.tell() if space: try: to_write = read(space) except: self.abort() raise self._buffer.write(to_write) if len(to_write) < space: return # EOF or incomplete self.__flush_buffer() to_write = read(self.chunk_size) while to_write and len(to_write) == self.chunk_size: self.__flush_data(to_write) to_write = read(self.chunk_size) self._buffer.write(to_write) def writelines(self, sequence): """Write a sequence of strings to the file. Does not add seperators. """ for line in sequence: self.write(line) def __enter__(self): """Support for the context manager protocol. """ return self def __exit__(self, exc_type, exc_val, exc_tb): """Support for the context manager protocol. Close the file and allow exceptions to propagate. """ self.close() # propagate exceptions return False class GridOut(object): """Class to read data out of GridFS. """ def __init__(self, root_collection, file_id=None, file_document=None, session=None): """Read a file from GridFS Application developers should generally not need to instantiate this class directly - instead see the methods provided by :class:`~gridfs.GridFS`. Either `file_id` or `file_document` must be specified, `file_document` will be given priority if present. Raises :class:`TypeError` if `root_collection` is not an instance of :class:`~pymongo.collection.Collection`. :Parameters: - `root_collection`: root collection to read from - `file_id` (optional): value of ``"_id"`` for the file to read - `file_document` (optional): file document from `root_collection.files` - `session` (optional): a :class:`~pymongo.client_session.ClientSession` to use for all commands .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.0 Creating a GridOut does not immediately retrieve the file metadata from the server. Metadata is fetched when first needed. """ if not isinstance(root_collection, Collection): raise TypeError("root_collection must be an " "instance of Collection") self.__chunks = root_collection.chunks self.__files = root_collection.files self.__file_id = file_id self.__buffer = EMPTY self.__position = 0 self._file = file_document self._session = session _id = _grid_out_property("_id", "The ``'_id'`` value for this file.") filename = _grid_out_property("filename", "Name of this file.") name = _grid_out_property("filename", "Alias for `filename`.") content_type = _grid_out_property("contentType", "Mime-type for this file.") length = _grid_out_property("length", "Length (in bytes) of this file.") chunk_size = _grid_out_property("chunkSize", "Chunk size for this file.") upload_date = _grid_out_property("uploadDate", "Date that this file was first uploaded.") aliases = _grid_out_property("aliases", "List of aliases for this file.") metadata = _grid_out_property("metadata", "Metadata attached to this file.") md5 = _grid_out_property("md5", "MD5 of the contents of this file " "(generated on the server).") def _ensure_file(self): if not self._file: self._file = self.__files.find_one({"_id": self.__file_id}, session=self._session) if not self._file: raise NoFile("no file in gridfs collection %r with _id %r" % (self.__files, self.__file_id)) def __getattr__(self, name): self._ensure_file() if name in self._file: return self._file[name] raise AttributeError("GridOut object has no attribute '%s'" % name) def readchunk(self): """Reads a chunk at a time. If the current position is within a chunk the remainder of the chunk is returned. """ received = len(self.__buffer) chunk_data = EMPTY chunk_size = int(self.chunk_size) if received > 0: chunk_data = self.__buffer elif self.__position < int(self.length): chunk_number = int((received + self.__position) / chunk_size) chunk = self.__chunks.find_one({"files_id": self._id, "n": chunk_number}, session=self._session) if not chunk: raise CorruptGridFile("no chunk #%d" % chunk_number) chunk_data = chunk["data"][self.__position % chunk_size:] if not chunk_data: raise CorruptGridFile("truncated chunk") self.__position += len(chunk_data) self.__buffer = EMPTY return chunk_data def read(self, size=-1): """Read at most `size` bytes from the file (less if there isn't enough data). The bytes are returned as an instance of :class:`str` (:class:`bytes` in python 3). If `size` is negative or omitted all data is read. :Parameters: - `size` (optional): the number of bytes to read """ self._ensure_file() if size == 0: return EMPTY remainder = int(self.length) - self.__position if size < 0 or size > remainder: size = remainder received = 0 data = StringIO() while received < size: chunk_data = self.readchunk() received += len(chunk_data) data.write(chunk_data) # Detect extra chunks. max_chunk_n = math.ceil(self.length / float(self.chunk_size)) chunk = self.__chunks.find_one({"files_id": self._id, "n": {"$gte": max_chunk_n}}, session=self._session) # According to spec, ignore extra chunks if they are empty. if chunk is not None and len(chunk['data']): raise CorruptGridFile( "Extra chunk found: expected %i chunks but found " "chunk with n=%i" % (max_chunk_n, chunk['n'])) self.__position -= received - size # Return 'size' bytes and store the rest. data.seek(size) self.__buffer = data.read() data.seek(0) return data.read(size) def readline(self, size=-1): """Read one line or up to `size` bytes from the file. :Parameters: - `size` (optional): the maximum number of bytes to read """ if size == 0: return b'' remainder = int(self.length) - self.__position if size < 0 or size > remainder: size = remainder received = 0 data = StringIO() while received < size: chunk_data = self.readchunk() pos = chunk_data.find(NEWLN, 0, size) if pos != -1: size = received + pos + 1 received += len(chunk_data) data.write(chunk_data) if pos != -1: break self.__position -= received - size # Return 'size' bytes and store the rest. data.seek(size) self.__buffer = data.read() data.seek(0) return data.read(size) def tell(self): """Return the current position of this file. """ return self.__position def seek(self, pos, whence=_SEEK_SET): """Set the current position of this file. :Parameters: - `pos`: the position (or offset if using relative positioning) to seek to - `whence` (optional): where to seek from. :attr:`os.SEEK_SET` (``0``) for absolute file positioning, :attr:`os.SEEK_CUR` (``1``) to seek relative to the current position, :attr:`os.SEEK_END` (``2``) to seek relative to the file's end. """ if whence == _SEEK_SET: new_pos = pos elif whence == _SEEK_CUR: new_pos = self.__position + pos elif whence == _SEEK_END: new_pos = int(self.length) + pos else: raise IOError(22, "Invalid value for `whence`") if new_pos < 0: raise IOError(22, "Invalid value for `pos` - must be positive") self.__position = new_pos self.__buffer = EMPTY def __iter__(self): """Return an iterator over all of this file's data. The iterator will return chunk-sized instances of :class:`str` (:class:`bytes` in python 3). This can be useful when serving files using a webserver that handles such an iterator efficiently. """ return GridOutIterator(self, self.__chunks, self._session) def close(self): """Make GridOut more generically file-like.""" pass def __enter__(self): """Makes it possible to use :class:`GridOut` files with the context manager protocol. """ return self def __exit__(self, exc_type, exc_val, exc_tb): """Makes it possible to use :class:`GridOut` files with the context manager protocol. """ return False class GridOutIterator(object): def __init__(self, grid_out, chunks, session): self.__id = grid_out._id self.__chunks = chunks self.__session = session self.__current_chunk = 0 self.__max_chunk = math.ceil(float(grid_out.length) / grid_out.chunk_size) def __iter__(self): return self def next(self): if self.__current_chunk >= self.__max_chunk: raise StopIteration chunk = self.__chunks.find_one({"files_id": self.__id, "n": self.__current_chunk}, session=self.__session) if not chunk: raise CorruptGridFile("no chunk #%d" % self.__current_chunk) self.__current_chunk += 1 return bytes(chunk["data"]) __next__ = next class GridOutCursor(Cursor): """A cursor / iterator for returning GridOut objects as the result of an arbitrary query against the GridFS files collection. """ def __init__(self, collection, filter=None, skip=0, limit=0, no_cursor_timeout=False, sort=None, batch_size=0, session=None): """Create a new cursor, similar to the normal :class:`~pymongo.cursor.Cursor`. Should not be called directly by application developers - see the :class:`~gridfs.GridFS` method :meth:`~gridfs.GridFS.find` instead. .. versionadded 2.7 .. mongodoc:: cursors """ # Hold on to the base "fs" collection to create GridOut objects later. self.__root_collection = collection super(GridOutCursor, self).__init__( collection.files, filter, skip=skip, limit=limit, no_cursor_timeout=no_cursor_timeout, sort=sort, batch_size=batch_size, session=session) def next(self): """Get next GridOut object from cursor. """ # Work around "super is not iterable" issue in Python 3.x next_file = super(GridOutCursor, self).next() return GridOut(self.__root_collection, file_document=next_file, session=self.session) __next__ = next def add_option(self, *args, **kwargs): raise NotImplementedError("Method does not exist for GridOutCursor") def remove_option(self, *args, **kwargs): raise NotImplementedError("Method does not exist for GridOutCursor") def _clone_base(self, session): """Creates an empty GridOutCursor for information to be copied into. """ return GridOutCursor(self.__root_collection, session=session) pymongo-3.6.1/gridfs/__init__.py0000644000076600000240000010475413245621354017042 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """GridFS is a specification for storing large objects in Mongo. The :mod:`gridfs` package is an implementation of GridFS on top of :mod:`pymongo`, exposing a file-like interface. .. mongodoc:: gridfs """ from bson.py3compat import abc from gridfs.errors import NoFile from gridfs.grid_file import (GridIn, GridOut, GridOutCursor, DEFAULT_CHUNK_SIZE) from pymongo import (ASCENDING, DESCENDING) from pymongo.common import UNAUTHORIZED_CODES, validate_string from pymongo.database import Database from pymongo.errors import ConfigurationError, OperationFailure class GridFS(object): """An instance of GridFS on top of a single Database. """ def __init__(self, database, collection="fs"): """Create a new instance of :class:`GridFS`. Raises :class:`TypeError` if `database` is not an instance of :class:`~pymongo.database.Database`. :Parameters: - `database`: database to use - `collection` (optional): root collection to use .. versionchanged:: 3.1 Indexes are only ensured on the first write to the DB. .. versionchanged:: 3.0 `database` must use an acknowledged :attr:`~pymongo.database.Database.write_concern` .. mongodoc:: gridfs """ if not isinstance(database, Database): raise TypeError("database must be an instance of Database") if not database.write_concern.acknowledged: raise ConfigurationError('database must use ' 'acknowledged write_concern') self.__database = database self.__collection = database[collection] self.__files = self.__collection.files self.__chunks = self.__collection.chunks def new_file(self, **kwargs): """Create a new file in GridFS. Returns a new :class:`~gridfs.grid_file.GridIn` instance to which data can be written. Any keyword arguments will be passed through to :meth:`~gridfs.grid_file.GridIn`. If the ``"_id"`` of the file is manually specified, it must not already exist in GridFS. Otherwise :class:`~gridfs.errors.FileExists` is raised. :Parameters: - `**kwargs` (optional): keyword arguments for file creation """ # No need for __ensure_index_files_id() here; GridIn ensures # the (files_id, n) index when needed. return GridIn(self.__collection, **kwargs) def put(self, data, **kwargs): """Put data in GridFS as a new file. Equivalent to doing:: try: f = new_file(**kwargs) f.write(data) finally: f.close() `data` can be either an instance of :class:`str` (:class:`bytes` in python 3) or a file-like object providing a :meth:`read` method. If an `encoding` keyword argument is passed, `data` can also be a :class:`unicode` (:class:`str` in python 3) instance, which will be encoded as `encoding` before being written. Any keyword arguments will be passed through to the created file - see :meth:`~gridfs.grid_file.GridIn` for possible arguments. Returns the ``"_id"`` of the created file. If the ``"_id"`` of the file is manually specified, it must not already exist in GridFS. Otherwise :class:`~gridfs.errors.FileExists` is raised. :Parameters: - `data`: data to be written as a file. - `**kwargs` (optional): keyword arguments for file creation .. versionchanged:: 3.0 w=0 writes to GridFS are now prohibited. """ grid_file = GridIn(self.__collection, **kwargs) try: grid_file.write(data) finally: grid_file.close() return grid_file._id def get(self, file_id, session=None): """Get a file from GridFS by ``"_id"``. Returns an instance of :class:`~gridfs.grid_file.GridOut`, which provides a file-like interface for reading. :Parameters: - `file_id`: ``"_id"`` of the file to get - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ gout = GridOut(self.__collection, file_id, session=session) # Raise NoFile now, instead of on first attribute access. gout._ensure_file() return gout def get_version(self, filename=None, version=-1, session=None, **kwargs): """Get a file from GridFS by ``"filename"`` or metadata fields. Returns a version of the file in GridFS whose filename matches `filename` and whose metadata fields match the supplied keyword arguments, as an instance of :class:`~gridfs.grid_file.GridOut`. Version numbering is a convenience atop the GridFS API provided by MongoDB. If more than one file matches the query (either by `filename` alone, by metadata fields, or by a combination of both), then version ``-1`` will be the most recently uploaded matching file, ``-2`` the second most recently uploaded, etc. Version ``0`` will be the first version uploaded, ``1`` the second version, etc. So if three versions have been uploaded, then version ``0`` is the same as version ``-3``, version ``1`` is the same as version ``-2``, and version ``2`` is the same as version ``-1``. Raises :class:`~gridfs.errors.NoFile` if no such version of that file exists. :Parameters: - `filename`: ``"filename"`` of the file to get, or `None` - `version` (optional): version of the file to get (defaults to -1, the most recent version uploaded) - `session` (optional): a :class:`~pymongo.client_session.ClientSession` - `**kwargs` (optional): find files by custom metadata. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.1 ``get_version`` no longer ensures indexes. """ query = kwargs if filename is not None: query["filename"] = filename cursor = self.__files.find(query, session=session) if version < 0: skip = abs(version) - 1 cursor.limit(-1).skip(skip).sort("uploadDate", DESCENDING) else: cursor.limit(-1).skip(version).sort("uploadDate", ASCENDING) try: doc = next(cursor) return GridOut( self.__collection, file_document=doc, session=session) except StopIteration: raise NoFile("no version %d for filename %r" % (version, filename)) def get_last_version(self, filename=None, session=None, **kwargs): """Get the most recent version of a file in GridFS by ``"filename"`` or metadata fields. Equivalent to calling :meth:`get_version` with the default `version` (``-1``). :Parameters: - `filename`: ``"filename"`` of the file to get, or `None` - `session` (optional): a :class:`~pymongo.client_session.ClientSession` - `**kwargs` (optional): find files by custom metadata. .. versionchanged:: 3.6 Added ``session`` parameter. """ return self.get_version(filename=filename, session=session, **kwargs) # TODO add optional safe mode for chunk removal? def delete(self, file_id, session=None): """Delete a file from GridFS by ``"_id"``. Deletes all data belonging to the file with ``"_id"``: `file_id`. .. warning:: Any processes/threads reading from the file while this method is executing will likely see an invalid/corrupt file. Care should be taken to avoid concurrent reads to a file while it is being deleted. .. note:: Deletes of non-existent files are considered successful since the end result is the same: no file with that _id remains. :Parameters: - `file_id`: ``"_id"`` of the file to delete - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.1 ``delete`` no longer ensures indexes. """ self.__files.delete_one({"_id": file_id}, session=session) self.__chunks.delete_many({"files_id": file_id}, session=session) def list(self, session=None): """List the names of all files stored in this instance of :class:`GridFS`. :Parameters: - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.1 ``list`` no longer ensures indexes. """ # With an index, distinct includes documents with no filename # as None. return [ name for name in self.__files.distinct("filename", session=session) if name is not None] def find_one(self, filter=None, session=None, *args, **kwargs): """Get a single file from gridfs. All arguments to :meth:`find` are also valid arguments for :meth:`find_one`, although any `limit` argument will be ignored. Returns a single :class:`~gridfs.grid_file.GridOut`, or ``None`` if no matching file is found. For example:: file = fs.find_one({"filename": "lisa.txt"}) :Parameters: - `filter` (optional): a dictionary specifying the query to be performing OR any other type to be used as the value for a query for ``"_id"`` in the file collection. - `*args` (optional): any additional positional arguments are the same as the arguments to :meth:`find`. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`find`. .. versionchanged:: 3.6 Added ``session`` parameter. """ if filter is not None and not isinstance(filter, abc.Mapping): filter = {"_id": filter} for f in self.find(filter, *args, session=session, **kwargs): return f return None def find(self, *args, **kwargs): """Query GridFS for files. Returns a cursor that iterates across files matching arbitrary queries on the files collection. Can be combined with other modifiers for additional control. For example:: for grid_out in fs.find({"filename": "lisa.txt"}, no_cursor_timeout=True): data = grid_out.read() would iterate through all versions of "lisa.txt" stored in GridFS. Note that setting no_cursor_timeout to True may be important to prevent the cursor from timing out during long multi-file processing work. As another example, the call:: most_recent_three = fs.find().sort("uploadDate", -1).limit(3) would return a cursor to the three most recently uploaded files in GridFS. Follows a similar interface to :meth:`~pymongo.collection.Collection.find` in :class:`~pymongo.collection.Collection`. If a :class:`~pymongo.client_session.ClientSession` is passed to :meth:`find`, all returned :class:`~gridfs.grid_file.GridOut` instances are associated with that session. :Parameters: - `filter` (optional): a SON object specifying elements which must be present for a document to be included in the result set - `skip` (optional): the number of files to omit (from the start of the result set) when returning the results - `limit` (optional): the maximum number of results to return - `no_cursor_timeout` (optional): if False (the default), any returned cursor is closed by the server after 10 minutes of inactivity. If set to True, the returned cursor will never time out on the server. Care should be taken to ensure that cursors with no_cursor_timeout turned on are properly closed. - `sort` (optional): a list of (key, direction) pairs specifying the sort order for this query. See :meth:`~pymongo.cursor.Cursor.sort` for details. Raises :class:`TypeError` if any of the arguments are of improper type. Returns an instance of :class:`~gridfs.grid_file.GridOutCursor` corresponding to this query. .. versionchanged:: 3.0 Removed the read_preference, tag_sets, and secondary_acceptable_latency_ms options. .. versionadded:: 2.7 .. mongodoc:: find """ return GridOutCursor(self.__collection, *args, **kwargs) def exists(self, document_or_id=None, session=None, **kwargs): """Check if a file exists in this instance of :class:`GridFS`. The file to check for can be specified by the value of its ``_id`` key, or by passing in a query document. A query document can be passed in as dictionary, or by using keyword arguments. Thus, the following three calls are equivalent: >>> fs.exists(file_id) >>> fs.exists({"_id": file_id}) >>> fs.exists(_id=file_id) As are the following two calls: >>> fs.exists({"filename": "mike.txt"}) >>> fs.exists(filename="mike.txt") And the following two: >>> fs.exists({"foo": {"$gt": 12}}) >>> fs.exists(foo={"$gt": 12}) Returns ``True`` if a matching file exists, ``False`` otherwise. Calls to :meth:`exists` will not automatically create appropriate indexes; application developers should be sure to create indexes if needed and as appropriate. :Parameters: - `document_or_id` (optional): query document, or _id of the document to check for - `session` (optional): a :class:`~pymongo.client_session.ClientSession` - `**kwargs` (optional): keyword arguments are used as a query document, if they're present. .. versionchanged:: 3.6 Added ``session`` parameter. """ if kwargs: f = self.__files.find_one(kwargs, ["_id"], session=session) else: f = self.__files.find_one(document_or_id, ["_id"], session=session) return f is not None class GridFSBucket(object): """An instance of GridFS on top of a single Database.""" def __init__(self, db, bucket_name="fs", chunk_size_bytes=DEFAULT_CHUNK_SIZE, write_concern=None, read_preference=None): """Create a new instance of :class:`GridFSBucket`. Raises :exc:`TypeError` if `database` is not an instance of :class:`~pymongo.database.Database`. Raises :exc:`~pymongo.errors.ConfigurationError` if `write_concern` is not acknowledged. :Parameters: - `database`: database to use. - `bucket_name` (optional): The name of the bucket. Defaults to 'fs'. - `chunk_size_bytes` (optional): The chunk size in bytes. Defaults to 255KB. - `write_concern` (optional): The :class:`~pymongo.write_concern.WriteConcern` to use. If ``None`` (the default) db.write_concern is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) db.read_preference is used. .. versionadded:: 3.1 .. mongodoc:: gridfs """ if not isinstance(db, Database): raise TypeError("database must be an instance of Database") wtc = write_concern if write_concern is not None else db.write_concern if not wtc.acknowledged: raise ConfigurationError('write concern must be acknowledged') self._db = db self._bucket_name = bucket_name self._collection = db[bucket_name] self._chunks = self._collection.chunks.with_options( write_concern=write_concern, read_preference=read_preference) self._files = self._collection.files.with_options( write_concern=write_concern, read_preference=read_preference) self._chunk_size_bytes = chunk_size_bytes def open_upload_stream(self, filename, chunk_size_bytes=None, metadata=None, session=None): """Opens a Stream that the application can write the contents of the file to. The user must specify the filename, and can choose to add any additional information in the metadata field of the file document or modify the chunk size. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) grid_in, file_id = fs.open_upload_stream( "test_file", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) grid_in.write("data I want to store!") grid_in.close() # uploaded on close Returns an instance of :class:`~gridfs.grid_file.GridIn`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to upload. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes in :class:`GridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ validate_string("filename", filename) opts = {"filename": filename, "chunk_size": (chunk_size_bytes if chunk_size_bytes is not None else self._chunk_size_bytes)} if metadata is not None: opts["metadata"] = metadata return GridIn(self._collection, session=session, **opts) def open_upload_stream_with_id( self, file_id, filename, chunk_size_bytes=None, metadata=None, session=None): """Opens a Stream that the application can write the contents of the file to. The user must specify the file id and filename, and can choose to add any additional information in the metadata field of the file document or modify the chunk size. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) grid_in, file_id = fs.open_upload_stream( ObjectId(), "test_file", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) grid_in.write("data I want to store!") grid_in.close() # uploaded on close Returns an instance of :class:`~gridfs.grid_file.GridIn`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `file_id`: The id to use for this file. The id must not have already been used for another file. - `filename`: The name of the file to upload. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes in :class:`GridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ validate_string("filename", filename) opts = {"_id": file_id, "filename": filename, "chunk_size": (chunk_size_bytes if chunk_size_bytes is not None else self._chunk_size_bytes)} if metadata is not None: opts["metadata"] = metadata return GridIn(self._collection, session=session, **opts) def upload_from_stream(self, filename, source, chunk_size_bytes=None, metadata=None, session=None): """Uploads a user file to a GridFS bucket. Reads the contents of the user file from `source` and uploads it to the file `filename`. Source can be a string or file-like object. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) file_id = fs.upload_from_stream( "test_file", "data I want to store!", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) Returns the _id of the uploaded file. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to upload. - `source`: The source stream of the content to be uploaded. Must be a file-like object that implements :meth:`read` or a string. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes of :class:`GridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ with self.open_upload_stream( filename, chunk_size_bytes, metadata, session=session) as gin: gin.write(source) return gin._id def upload_from_stream_with_id(self, file_id, filename, source, chunk_size_bytes=None, metadata=None, session=None): """Uploads a user file to a GridFS bucket with a custom file id. Reads the contents of the user file from `source` and uploads it to the file `filename`. Source can be a string or file-like object. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) file_id = fs.upload_from_stream( ObjectId(), "test_file", "data I want to store!", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `file_id`: The id to use for this file. The id must not have already been used for another file. - `filename`: The name of the file to upload. - `source`: The source stream of the content to be uploaded. Must be a file-like object that implements :meth:`read` or a string. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes of :class:`GridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ with self.open_upload_stream_with_id( file_id, filename, chunk_size_bytes, metadata, session=session) as gin: gin.write(source) def open_download_stream(self, file_id, session=None): """Opens a Stream from which the application can read the contents of the stored file specified by file_id. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # get _id of file to read. file_id = fs.upload_from_stream("test_file", "data I want to store!") grid_out = fs.open_download_stream(file_id) contents = grid_out.read() Returns an instance of :class:`~gridfs.grid_file.GridOut`. Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be downloaded. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ gout = GridOut(self._collection, file_id, session=session) # Raise NoFile now, instead of on first attribute access. gout._ensure_file() return gout def download_to_stream(self, file_id, destination, session=None): """Downloads the contents of the stored file specified by file_id and writes the contents to `destination`. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # Get _id of file to read file_id = fs.upload_from_stream("test_file", "data I want to store!") # Get file to write to file = open('myfile','wb+') fs.download_to_stream(file_id, file) file.seek(0) contents = file.read() Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be downloaded. - `destination`: a file-like object implementing :meth:`write`. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ gout = self.open_download_stream(file_id, session=session) for chunk in gout: destination.write(chunk) def delete(self, file_id, session=None): """Given an file_id, delete this stored file's files collection document and associated chunks from a GridFS bucket. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # Get _id of file to delete file_id = fs.upload_from_stream("test_file", "data I want to store!") fs.delete(file_id) Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be deleted. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ res = self._files.delete_one({"_id": file_id}, session=session) self._chunks.delete_many({"files_id": file_id}, session=session) if not res.deleted_count: raise NoFile( "no file could be deleted because none matched %s" % file_id) def find(self, *args, **kwargs): """Find and return the files collection documents that match ``filter`` Returns a cursor that iterates across files matching arbitrary queries on the files collection. Can be combined with other modifiers for additional control. For example:: for grid_data in fs.find({"filename": "lisa.txt"}, no_cursor_timeout=True): data = grid_data.read() would iterate through all versions of "lisa.txt" stored in GridFS. Note that setting no_cursor_timeout to True may be important to prevent the cursor from timing out during long multi-file processing work. As another example, the call:: most_recent_three = fs.find().sort("uploadDate", -1).limit(3) would return a cursor to the three most recently uploaded files in GridFS. Follows a similar interface to :meth:`~pymongo.collection.Collection.find` in :class:`~pymongo.collection.Collection`. If a :class:`~pymongo.client_session.ClientSession` is passed to :meth:`find`, all returned :class:`~gridfs.grid_file.GridOut` instances are associated with that session. :Parameters: - `filter`: Search query. - `batch_size` (optional): The number of documents to return per batch. - `limit` (optional): The maximum number of documents to return. - `no_cursor_timeout` (optional): The server normally times out idle cursors after an inactivity period (10 minutes) to prevent excess memory use. Set this option to True prevent that. - `skip` (optional): The number of documents to skip before returning. - `sort` (optional): The order by which to sort results. Defaults to None. """ return GridOutCursor(self._collection, *args, **kwargs) def open_download_stream_by_name(self, filename, revision=-1, session=None): """Opens a Stream from which the application can read the contents of `filename` and optional `revision`. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) grid_out = fs.open_download_stream_by_name("test_file") contents = grid_out.read() Returns an instance of :class:`~gridfs.grid_file.GridOut`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` filename is not a string. :Parameters: - `filename`: The name of the file to read from. - `revision` (optional): Which revision (documents with the same filename and different uploadDate) of the file to retrieve. Defaults to -1 (the most recent revision). - `session` (optional): a :class:`~pymongo.client_session.ClientSession` :Note: Revision numbers are defined as follows: - 0 = the original stored file - 1 = the first revision - 2 = the second revision - etc... - -2 = the second most recent revision - -1 = the most recent revision .. versionchanged:: 3.6 Added ``session`` parameter. """ validate_string("filename", filename) query = {"filename": filename} cursor = self._files.find(query, session=session) if revision < 0: skip = abs(revision) - 1 cursor.limit(-1).skip(skip).sort("uploadDate", DESCENDING) else: cursor.limit(-1).skip(revision).sort("uploadDate", ASCENDING) try: grid_file = next(cursor) return GridOut( self._collection, file_document=grid_file, session=session) except StopIteration: raise NoFile( "no version %d for filename %r" % (revision, filename)) def download_to_stream_by_name(self, filename, destination, revision=-1, session=None): """Write the contents of `filename` (with optional `revision`) to `destination`. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # Get file to write to file = open('myfile','wb') fs.download_to_stream_by_name("test_file", file) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to read from. - `destination`: A file-like object that implements :meth:`write`. - `revision` (optional): Which revision (documents with the same filename and different uploadDate) of the file to retrieve. Defaults to -1 (the most recent revision). - `session` (optional): a :class:`~pymongo.client_session.ClientSession` :Note: Revision numbers are defined as follows: - 0 = the original stored file - 1 = the first revision - 2 = the second revision - etc... - -2 = the second most recent revision - -1 = the most recent revision .. versionchanged:: 3.6 Added ``session`` parameter. """ gout = self.open_download_stream_by_name( filename, revision, session=session) for chunk in gout: destination.write(chunk) def rename(self, file_id, new_filename, session=None): """Renames the stored file with the specified file_id. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # Get _id of file to rename file_id = fs.upload_from_stream("test_file", "data I want to store!") fs.rename(file_id, "new_test_name") Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be renamed. - `new_filename`: The new name of the file. - `session` (optional): a :class:`~pymongo.client_session.ClientSession` .. versionchanged:: 3.6 Added ``session`` parameter. """ result = self._files.update_one({"_id": file_id}, {"$set": {"filename": new_filename}}, session=session) if not result.matched_count: raise NoFile("no files could be renamed %r because none " "matched file_id %i" % (new_filename, file_id)) pymongo-3.6.1/gridfs/errors.py0000644000076600000240000000204013245617773016612 0ustar shanestaff00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Exceptions raised by the :mod:`gridfs` package""" from pymongo.errors import PyMongoError class GridFSError(PyMongoError): """Base class for all GridFS exceptions.""" class CorruptGridFile(GridFSError): """Raised when a file in :class:`~gridfs.GridFS` is malformed.""" class NoFile(GridFSError): """Raised when trying to read from a non-existent file.""" class FileExists(GridFSError): """Raised when trying to create a file that already exists.""" pymongo-3.6.1/pymongo.egg-info/0000755000076600000240000000000013246104133016612 5ustar shanestaff00000000000000pymongo-3.6.1/pymongo.egg-info/PKG-INFO0000644000076600000240000002201513246104132017706 0ustar shanestaff00000000000000Metadata-Version: 1.1 Name: pymongo Version: 3.6.1 Summary: Python driver for MongoDB Home-page: http://github.com/mongodb/mongo-python-driver Author: Bernie Hackett Author-email: bernie@mongodb.com License: Apache License, Version 2.0 Description: ======= PyMongo ======= :Info: See `the mongo site `_ for more information. See `github `_ for the latest source. :Author: Mike Dirolf :Maintainer: Bernie Hackett About ===== The PyMongo distribution contains tools for interacting with MongoDB database from Python. The ``bson`` package is an implementation of the `BSON format `_ for Python. The ``pymongo`` package is a native Python driver for MongoDB. The ``gridfs`` package is a `gridfs `_ implementation on top of ``pymongo``. PyMongo supports MongoDB 2.6, 3.0, 3.2, 3.4, and 3.6. Support / Feedback ================== For issues with, questions about, or feedback for PyMongo, please look into our `support channels `_. Please do not email any of the PyMongo developers directly with issues or questions - you're more likely to get an answer on the `mongodb-user `_ list on Google Groups. Bugs / Feature Requests ======================= Think you’ve found a bug? Want to see a new feature in PyMongo? Please open a case in our issue management tool, JIRA: - `Create an account and login `_. - Navigate to `the PYTHON project `_. - Click **Create Issue** - Please provide as much information as possible about the issue type and how to reproduce it. Bug reports in JIRA for all driver projects (i.e. PYTHON, CSHARP, JAVA) and the Core Server (i.e. SERVER) project are **public**. How To Ask For Help ------------------- Please include all of the following information when opening an issue: - Detailed steps to reproduce the problem, including full traceback, if possible. - The exact python version used, with patch level:: $ python -c "import sys; print(sys.version)" - The exact version of PyMongo used, with patch level:: $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())" - The operating system and version (e.g. Windows 7, OSX 10.8, ...) - Web framework or asynchronous network library used, if any, with version (e.g. Django 1.7, mod_wsgi 4.3.0, gevent 1.0.1, Tornado 4.0.2, ...) Security Vulnerabilities ------------------------ If you’ve identified a security vulnerability in a driver or any other MongoDB project, please report it according to the `instructions here `_. Installation ============ PyMongo can be installed with `pip `_:: $ python -m pip install pymongo Or ``easy_install`` from `setuptools `_:: $ python -m easy_install pymongo You can also download the project source and do:: $ python setup.py install Do **not** install the "bson" package from pypi. PyMongo comes with its own bson package; doing "easy_install bson" installs a third-party package that is incompatible with PyMongo. Dependencies ============ PyMongo supports CPython 2.6, 2.7, 3.4+, PyPy, and PyPy3. Optional dependencies: GSSAPI authentication requires `pykerberos `_ on Unix or `WinKerberos `_ on Windows. The correct dependency can be installed automatically along with PyMongo:: $ python -m pip install pymongo[gssapi] Support for mongodb+srv:// URIs requires `dnspython `_:: $ python -m pip install pymongo[srv] TLS / SSL support may require `ipaddress `_ and `certifi `_ or `wincertstore `_ depending on the Python version in use. The necessary dependencies can be installed along with PyMongo:: $ python -m pip install pymongo[tls] You can install all dependencies automatically with the following command:: $ python -m pip install pymongo[gssapi,srv,tls] Other optional packages: - `backports.pbkdf2 `_, improves authentication performance with SCRAM-SHA-1, the default authentication mechanism for MongoDB 3.0+. It especially improves performance on Python versions older than 2.7.8. - `monotonic `_ adds support for a monotonic clock, which improves reliability in environments where clock adjustments are frequent. Not needed in Python 3. Additional dependencies are: - (to generate documentation) sphinx_ - (to run the tests under Python 2.6) unittest2_ Examples ======== Here's a basic example (for more see the *examples* section of the docs): .. code-block:: python >>> import pymongo >>> client = pymongo.MongoClient("localhost", 27017) >>> db = client.test >>> db.name u'test' >>> db.my_collection Collection(Database(MongoClient('localhost', 27017), u'test'), u'my_collection') >>> db.my_collection.insert_one({"x": 10}).inserted_id ObjectId('4aba15ebe23f6b53b0000000') >>> db.my_collection.insert_one({"x": 8}).inserted_id ObjectId('4aba160ee23f6b543e000000') >>> db.my_collection.insert_one({"x": 11}).inserted_id ObjectId('4aba160ee23f6b543e000002') >>> db.my_collection.find_one() {u'x': 10, u'_id': ObjectId('4aba15ebe23f6b53b0000000')} >>> for item in db.my_collection.find(): ... print(item["x"]) ... 10 8 11 >>> db.my_collection.create_index("x") u'x_1' >>> for item in db.my_collection.find().sort("x", pymongo.ASCENDING): ... print(item["x"]) ... 8 10 11 >>> [item["x"] for item in db.my_collection.find().limit(2).skip(1)] [8, 11] Documentation ============= You will need sphinx_ installed to generate the documentation. Documentation can be generated by running **python setup.py doc**. Generated documentation can be found in the *doc/build/html/* directory. Testing ======= The easiest way to run the tests is to run **python setup.py test** in the root of the distribution. Note that you will need unittest2_ to run the tests under Python 2.6. To verify that PyMongo works with Gevent's monkey-patching:: $ python green_framework_test.py gevent Or with Eventlet's:: $ python green_framework_test.py eventlet .. _sphinx: http://sphinx.pocoo.org/ .. _unittest2: https://pypi.python.org/pypi/unittest2 Keywords: mongo,mongodb,pymongo,gridfs,bson Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Apache Software License Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: POSIX Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Database pymongo-3.6.1/pymongo.egg-info/SOURCES.txt0000644000076600000240000002375113246104133020506 0ustar shanestaff00000000000000LICENSE MANIFEST.in README.rst THIRD-PARTY-NOTICES ez_setup.py setup.cfg setup.py bson/__init__.py bson/_cbsonmodule.c bson/_cbsonmodule.h bson/binary.py bson/bson-endian.h bson/bson-stdint-win32.h bson/buffer.c bson/buffer.h bson/code.py bson/codec_options.py bson/dbref.py bson/decimal128.py bson/encoding_helpers.c bson/encoding_helpers.h bson/errors.py bson/int64.py bson/json_util.py bson/max_key.py bson/min_key.py bson/objectid.py bson/py3compat.py bson/raw_bson.py bson/regex.py bson/son.py bson/time64.c bson/time64.h bson/time64_config.h bson/time64_limits.h bson/timestamp.py bson/tz_util.py doc/__init__.py doc/atlas.rst doc/changelog.rst doc/compatibility-policy.rst doc/conf.py doc/contributors.rst doc/faq.rst doc/index.rst doc/installation.rst doc/migrate-to-pymongo3.rst doc/mongo_extensions.py doc/python3.rst doc/tools.rst doc/tutorial.rst doc/_build/3.6.0/searchindex.js doc/_build/3.6.0/_images/periodic-executor-refs.png doc/_build/3.6.0/_static/basic.css doc/_build/3.6.0/_static/classic.css doc/_build/3.6.0/_static/comment-bright.png doc/_build/3.6.0/_static/comment-close.png doc/_build/3.6.0/_static/comment.png doc/_build/3.6.0/_static/default.css doc/_build/3.6.0/_static/doctools.js doc/_build/3.6.0/_static/down-pressed.png doc/_build/3.6.0/_static/down.png doc/_build/3.6.0/_static/file.png doc/_build/3.6.0/_static/jquery-3.1.0.js doc/_build/3.6.0/_static/jquery.js doc/_build/3.6.0/_static/minus.png doc/_build/3.6.0/_static/periodic-executor-refs.png doc/_build/3.6.0/_static/plus.png doc/_build/3.6.0/_static/pydoctheme.css doc/_build/3.6.0/_static/pygments.css doc/_build/3.6.0/_static/searchtools.js doc/_build/3.6.0/_static/sidebar.js doc/_build/3.6.0/_static/underscore-1.3.1.js doc/_build/3.6.0/_static/underscore.js doc/_build/3.6.0/_static/up-pressed.png doc/_build/3.6.0/_static/up.png doc/_build/3.6.0/_static/websupport.js doc/_build/3.6.1/searchindex.js doc/_build/3.6.1.dev0/searchindex.js doc/_build/3.6.1.dev0/_images/periodic-executor-refs.png doc/_build/3.6.1.dev0/_static/basic.css doc/_build/3.6.1.dev0/_static/classic.css doc/_build/3.6.1.dev0/_static/comment-bright.png doc/_build/3.6.1.dev0/_static/comment-close.png doc/_build/3.6.1.dev0/_static/comment.png doc/_build/3.6.1.dev0/_static/default.css doc/_build/3.6.1.dev0/_static/doctools.js doc/_build/3.6.1.dev0/_static/down-pressed.png doc/_build/3.6.1.dev0/_static/down.png doc/_build/3.6.1.dev0/_static/file.png doc/_build/3.6.1.dev0/_static/jquery-3.1.0.js doc/_build/3.6.1.dev0/_static/jquery.js doc/_build/3.6.1.dev0/_static/minus.png doc/_build/3.6.1.dev0/_static/periodic-executor-refs.png doc/_build/3.6.1.dev0/_static/plus.png doc/_build/3.6.1.dev0/_static/pydoctheme.css doc/_build/3.6.1.dev0/_static/pygments.css doc/_build/3.6.1.dev0/_static/searchtools.js doc/_build/3.6.1.dev0/_static/sidebar.js doc/_build/3.6.1.dev0/_static/underscore-1.3.1.js doc/_build/3.6.1.dev0/_static/underscore.js doc/_build/3.6.1.dev0/_static/up-pressed.png doc/_build/3.6.1.dev0/_static/up.png doc/_build/3.6.1.dev0/_static/websupport.js doc/_build/3.6.1/_images/periodic-executor-refs.png doc/_build/3.6.1/_static/basic.css doc/_build/3.6.1/_static/classic.css doc/_build/3.6.1/_static/comment-bright.png doc/_build/3.6.1/_static/comment-close.png doc/_build/3.6.1/_static/comment.png doc/_build/3.6.1/_static/default.css doc/_build/3.6.1/_static/doctools.js doc/_build/3.6.1/_static/down-pressed.png doc/_build/3.6.1/_static/down.png doc/_build/3.6.1/_static/file.png doc/_build/3.6.1/_static/jquery-3.1.0.js doc/_build/3.6.1/_static/jquery.js doc/_build/3.6.1/_static/minus.png doc/_build/3.6.1/_static/periodic-executor-refs.png doc/_build/3.6.1/_static/plus.png doc/_build/3.6.1/_static/pydoctheme.css doc/_build/3.6.1/_static/pygments.css doc/_build/3.6.1/_static/searchtools.js doc/_build/3.6.1/_static/sidebar.js doc/_build/3.6.1/_static/underscore-1.3.1.js doc/_build/3.6.1/_static/underscore.js doc/_build/3.6.1/_static/up-pressed.png doc/_build/3.6.1/_static/up.png doc/_build/3.6.1/_static/websupport.js doc/_build/3.6rc1.dev0/searchindex.js doc/_build/3.6rc1.dev0/_images/periodic-executor-refs.png doc/_build/3.6rc1.dev0/_static/basic.css doc/_build/3.6rc1.dev0/_static/classic.css doc/_build/3.6rc1.dev0/_static/comment-bright.png doc/_build/3.6rc1.dev0/_static/comment-close.png doc/_build/3.6rc1.dev0/_static/comment.png doc/_build/3.6rc1.dev0/_static/default.css doc/_build/3.6rc1.dev0/_static/doctools.js doc/_build/3.6rc1.dev0/_static/down-pressed.png doc/_build/3.6rc1.dev0/_static/down.png doc/_build/3.6rc1.dev0/_static/file.png doc/_build/3.6rc1.dev0/_static/jquery-3.1.0.js doc/_build/3.6rc1.dev0/_static/jquery.js doc/_build/3.6rc1.dev0/_static/minus.png doc/_build/3.6rc1.dev0/_static/periodic-executor-refs.png doc/_build/3.6rc1.dev0/_static/plus.png doc/_build/3.6rc1.dev0/_static/pydoctheme.css doc/_build/3.6rc1.dev0/_static/pygments.css doc/_build/3.6rc1.dev0/_static/searchtools.js doc/_build/3.6rc1.dev0/_static/sidebar.js doc/_build/3.6rc1.dev0/_static/underscore-1.3.1.js doc/_build/3.6rc1.dev0/_static/underscore.js doc/_build/3.6rc1.dev0/_static/up-pressed.png doc/_build/3.6rc1.dev0/_static/up.png doc/_build/3.6rc1.dev0/_static/websupport.js doc/api/index.rst doc/api/bson/binary.rst doc/api/bson/code.rst doc/api/bson/codec_options.rst doc/api/bson/dbref.rst doc/api/bson/decimal128.rst doc/api/bson/errors.rst doc/api/bson/index.rst doc/api/bson/int64.rst doc/api/bson/json_util.rst doc/api/bson/max_key.rst doc/api/bson/min_key.rst doc/api/bson/objectid.rst doc/api/bson/raw_bson.rst doc/api/bson/regex.rst doc/api/bson/son.rst doc/api/bson/timestamp.rst doc/api/bson/tz_util.rst doc/api/gridfs/errors.rst doc/api/gridfs/grid_file.rst doc/api/gridfs/index.rst doc/api/pymongo/bulk.rst doc/api/pymongo/change_stream.rst doc/api/pymongo/client_session.rst doc/api/pymongo/collation.rst doc/api/pymongo/collection.rst doc/api/pymongo/command_cursor.rst doc/api/pymongo/cursor.rst doc/api/pymongo/cursor_manager.rst doc/api/pymongo/database.rst doc/api/pymongo/errors.rst doc/api/pymongo/index.rst doc/api/pymongo/ismaster.rst doc/api/pymongo/message.rst doc/api/pymongo/mongo_client.rst doc/api/pymongo/mongo_replica_set_client.rst doc/api/pymongo/monitoring.rst doc/api/pymongo/operations.rst doc/api/pymongo/pool.rst doc/api/pymongo/read_concern.rst doc/api/pymongo/read_preferences.rst doc/api/pymongo/results.rst doc/api/pymongo/server_description.rst doc/api/pymongo/son_manipulator.rst doc/api/pymongo/topology_description.rst doc/api/pymongo/uri_parser.rst doc/api/pymongo/write_concern.rst doc/developer/index.rst doc/developer/periodic_executor.rst doc/examples/aggregation.rst doc/examples/authentication.rst doc/examples/bulk.rst doc/examples/collations.rst doc/examples/copydb.rst doc/examples/datetimes.rst doc/examples/geo.rst doc/examples/gevent.rst doc/examples/gridfs.rst doc/examples/high_availability.rst doc/examples/index.rst doc/examples/mod_wsgi.rst doc/examples/tailable.rst doc/examples/tls.rst doc/pydoctheme/theme.conf doc/pydoctheme/static/pydoctheme.css doc/static/periodic-executor-refs.png doc/static/sidebar.js gridfs/__init__.py gridfs/errors.py gridfs/grid_file.py pymongo/__init__.py pymongo/_cmessagemodule.c pymongo/auth.py pymongo/bulk.py pymongo/change_stream.py pymongo/client_options.py pymongo/client_session.py pymongo/collation.py pymongo/collection.py pymongo/command_cursor.py pymongo/common.py pymongo/cursor.py pymongo/cursor_manager.py pymongo/database.py pymongo/errors.py pymongo/helpers.py pymongo/ismaster.py pymongo/max_staleness_selectors.py pymongo/message.py pymongo/mongo_client.py pymongo/mongo_replica_set_client.py pymongo/monitor.py pymongo/monitoring.py pymongo/monotonic.py pymongo/network.py pymongo/operations.py pymongo/periodic_executor.py pymongo/pool.py pymongo/read_concern.py pymongo/read_preferences.py pymongo/response.py pymongo/results.py pymongo/server.py pymongo/server_description.py pymongo/server_selectors.py pymongo/server_type.py pymongo/settings.py pymongo/son_manipulator.py pymongo/ssl_context.py pymongo/ssl_match_hostname.py pymongo/ssl_support.py pymongo/thread_util.py pymongo/topology.py pymongo/topology_description.py pymongo/uri_parser.py pymongo/write_concern.py pymongo.egg-info/PKG-INFO pymongo.egg-info/SOURCES.txt pymongo.egg-info/dependency_links.txt pymongo.egg-info/requires.txt pymongo.egg-info/top_level.txt test/__init__.py test/pymongo_mocks.py test/qcheck.py test/test_auth.py test/test_binary.py test/test_bson.py test/test_bson_corpus.py test/test_bulk.py test/test_change_stream.py test/test_client.py test/test_client_context.py test/test_code.py test/test_collation.py test/test_collection.py test/test_command_monitoring_spec.py test/test_common.py test/test_crud.py test/test_cursor.py test/test_cursor_manager.py test/test_database.py test/test_dbref.py test/test_decimal128.py test/test_discovery_and_monitoring.py test/test_dns.py test/test_examples.py test/test_grid_file.py test/test_gridfs.py test/test_gridfs_bucket.py test/test_gridfs_spec.py test/test_heartbeat_monitoring.py test/test_json_util.py test/test_legacy_api.py test/test_max_staleness.py test/test_mongos_load_balancing.py test/test_monitor.py test/test_monitoring.py test/test_monotonic.py test/test_objectid.py test/test_pooling.py test/test_pymongo.py test/test_raw_bson.py test/test_read_concern.py test/test_read_preferences.py test/test_replica_set_client.py test/test_replica_set_reconfig.py test/test_retryable_writes.py test/test_sdam_monitoring_spec.py test/test_server.py test/test_server_description.py test/test_server_selection.py test/test_server_selection_rtt.py test/test_session.py test/test_son.py test/test_son_manipulator.py test/test_ssl.py test/test_threads.py test/test_timestamp.py test/test_topology.py test/test_uri_parser.py test/test_uri_spec.py test/utils.py test/utils_selection_tests.py test/version.py test/certificates/ca.pem test/certificates/client.pem test/certificates/client_encrypted.pem test/certificates/crl.pem test/certificates/server.pem test/high_availability/ha_tools.py test/high_availability/test_ha.py test/mod_wsgi_test/test_client.py test/performance/perf_test.py tools/README.rst tools/benchmark.py tools/clean.py tools/fail_if_no_c.pypymongo-3.6.1/pymongo.egg-info/requires.txt0000644000076600000240000000010513246104132021205 0ustar shanestaff00000000000000 [gssapi] pykerberos [srv] dnspython>=1.8.0,<2.0.0 [tls] ipaddress pymongo-3.6.1/pymongo.egg-info/top_level.txt0000644000076600000240000000002413246104132021337 0ustar shanestaff00000000000000bson gridfs pymongo pymongo-3.6.1/pymongo.egg-info/dependency_links.txt0000644000076600000240000000000113246104132022657 0ustar shanestaff00000000000000 pymongo-3.6.1/doc/0000755000076600000240000000000013246104133014175 5ustar shanestaff00000000000000pymongo-3.6.1/doc/index.rst0000644000076600000240000000555513245621354016060 0ustar shanestaff00000000000000PyMongo |release| Documentation =============================== Overview -------- **PyMongo** is a Python distribution containing tools for working with `MongoDB `_, and is the recommended way to work with MongoDB from Python. This documentation attempts to explain everything you need to know to use **PyMongo**. .. todo:: a list of PyMongo's features :doc:`installation` Instructions on how to get the distribution. :doc:`tutorial` Start here for a quick overview. :doc:`examples/index` Examples of how to perform specific tasks. :doc:`atlas` Using PyMongo with MongoDB Atlas. :doc:`examples/tls` Using PyMongo with TLS / SSL. :doc:`faq` Some questions that come up often. :doc:`migrate-to-pymongo3` A PyMongo 2.x to 3.x migration guide. :doc:`python3` Frequently asked questions about python 3 support. :doc:`compatibility-policy` Explanation of deprecations, and how to keep pace with changes in PyMongo's API. :doc:`api/index` The complete API documentation, organized by module. :doc:`tools` A listing of Python tools and libraries that have been written for MongoDB. :doc:`developer/index` Developer guide for contributors to PyMongo. Getting Help ------------ If you're having trouble or have questions about PyMongo, the best place to ask is the `MongoDB user group `_. Once you get an answer, it'd be great if you could work it back into this documentation and contribute! Issues ------ All issues should be reported (and can be tracked / voted for / commented on) at the main `MongoDB JIRA bug tracker `_, in the "Python Driver" project. Contributing ------------ **PyMongo** has a large :doc:`community ` and contributions are always encouraged. Contributions can be as simple as minor tweaks to this documentation. To contribute, fork the project on `github `_ and send a pull request. Changes ------- See the :doc:`changelog` for a full list of changes to PyMongo. For older versions of the documentation please see the `archive list `_. About This Documentation ------------------------ This documentation is generated using the `Sphinx `_ documentation generator. The source files for the documentation are located in the *doc/* directory of the **PyMongo** distribution. To generate the docs locally run the following command from the root directory of the **PyMongo** source: .. code-block:: bash $ python setup.py doc Indices and tables ------------------ * :ref:`genindex` * :ref:`modindex` * :ref:`search` .. toctree:: :hidden: atlas installation tutorial examples/index faq compatibility-policy api/index tools contributors changelog python3 migrate-to-pymongo3 developer/index pymongo-3.6.1/doc/developer/0000755000076600000240000000000013246104133016162 5ustar shanestaff00000000000000pymongo-3.6.1/doc/developer/index.rst0000644000076600000240000000020213245617773020037 0ustar shanestaff00000000000000Developer Guide =============== Technical guide for contributors to PyMongo. .. toctree:: :maxdepth: 1 periodic_executor pymongo-3.6.1/doc/developer/periodic_executor.rst0000644000076600000240000001223713245617773022457 0ustar shanestaff00000000000000Periodic Executors ================== .. currentmodule:: pymongo PyMongo implements a :class:`~periodic_executor.PeriodicExecutor` for two purposes: as the background thread for :class:`~monitor.Monitor`, and to regularly check if there are `OP_KILL_CURSORS` messages that must be sent to the server. Killing Cursors --------------- An incompletely iterated :class:`~cursor.Cursor` on the client represents an open cursor object on the server. In code like this, we lose a reference to the cursor before finishing iteration:: for doc in collection.find(): raise Exception() We try to send an `OP_KILL_CURSORS` to the server to tell it to clean up the server-side cursor. But we must not take any locks directly from the cursor's destructor (see `PYTHON-799`_), so we cannot safely use the PyMongo data structures required to send a message. The solution is to add the cursor's id to an array on the :class:`~mongo_client.MongoClient` without taking any locks. Each client has a :class:`~periodic_executor.PeriodicExecutor` devoted to checking the array for cursor ids. Any it sees are the result of cursors that were freed while the server-side cursor was still open. The executor can safely take the locks it needs in order to send the `OP_KILL_CURSORS` message. .. _PYTHON-799: https://jira.mongodb.org/browse/PYTHON-799 Stopping Executors ------------------ Just as :class:`~cursor.Cursor` must not take any locks from its destructor, neither can :class:`~mongo_client.MongoClient` and :class:`~topology.Topology`. Thus, although the client calls :meth:`close` on its kill-cursors thread, and the topology calls :meth:`close` on all its monitor threads, the :meth:`close` method cannot actually call :meth:`wake` on the executor, since :meth:`wake` takes a lock. Instead, executors wake periodically to check if ``self.close`` is set, and if so they exit. A thread can log spurious errors if it wakes late in the Python interpreter's shutdown sequence, so we try to join threads before then. Each periodic executor (either a monitor or a kill-cursors thread) adds a weakref to itself to a set called ``_EXECUTORS``, in the ``periodic_executor`` module. An `exit handler`_ runs on shutdown and tells all executors to stop, then tries (with a short timeout) to join all executor threads. .. _exit handler: https://docs.python.org/2/library/atexit.html Monitoring ---------- For each server in the topology, :class:`~topology.Topology` uses a periodic executor to launch a monitor thread. This thread must not prevent the topology from being freed, so it weakrefs the topology. Furthermore, it uses a weakref callback to terminate itself soon after the topology is freed. Solid lines represent strong references, dashed lines weak ones: .. generated with graphviz: "dot -Tpng periodic-executor-refs.dot > periodic-executor-refs.png" .. image:: ../static/periodic-executor-refs.png See `Stopping Executors`_ above for an explanation of the ``_EXECUTORS`` set. It is a requirement of the `Server Discovery And Monitoring Spec`_ that a sleeping monitor can be awakened early. Aside from infrequent wakeups to do their appointed chores, and occasional interruptions, periodic executors also wake periodically to check if they should terminate. Our first implementation of this idea was the obvious one: use the Python standard library's threading.Condition.wait with a timeout. Another thread wakes the executor early by signaling the condition variable. A topology cannot signal the condition variable to tell the executor to terminate, because it would risk a deadlock in the garbage collector: no destructor or weakref callback can take a lock to signal the condition variable (see `PYTHON-863`_); thus the only way for a dying object to terminate a periodic executor is to set its "stopped" flag and let the executor see the flag next time it wakes. We erred on the side of prompt cleanup, and set the check interval at 100ms. We assumed that checking a flag and going back to sleep 10 times a second was cheap on modern machines. Starting in Python 3.2, the builtin C implementation of lock.acquire takes a timeout parameter, so Python 3.2+ Condition variables sleep simply by calling lock.acquire; they are implemented as efficiently as expected. But in Python 2, lock.acquire has no timeout. To wait with a timeout, a Python 2 condition variable sleeps a millisecond, tries to acquire the lock, sleeps twice as long, and tries again. This exponential backoff reaches a maximum sleep time of 50ms. If PyMongo calls the condition variable's "wait" method with a short timeout, the exponential backoff is restarted frequently. Overall, the condition variable is not waking a few times a second, but hundreds of times. (See `PYTHON-983`_.) Thus the current design of periodic executors is surprisingly simple: they do a simple `time.sleep` for a half-second, check if it is time to wake or terminate, and sleep again. .. _Server Discovery And Monitoring Spec: https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/server-discovery-and-monitoring.rst#requesting-an-immediate-check .. _PYTHON-863: https://jira.mongodb.org/browse/PYTHON-863 .. _PYTHON-983: https://jira.mongodb.org/browse/PYTHON-983 pymongo-3.6.1/doc/conf.py0000644000076600000240000001232113245621354015503 0ustar shanestaff00000000000000# -*- coding: utf-8 -*- # # PyMongo documentation build configuration file # # This file is execfile()d with the current directory set to its containing dir. import sys, os sys.path[0:0] = [os.path.abspath('..')] import pymongo # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.coverage', 'sphinx.ext.todo', 'doc.mongo_extensions', 'sphinx.ext.intersphinx'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = u'PyMongo' copyright = u'MongoDB, Inc. 2008-present. MongoDB, Mongo, and the leaf logo are registered trademarks of MongoDB, Inc' html_show_sphinx = False # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = pymongo.version # The full version, including alpha/beta/rc tags. release = pymongo.version # List of documents that shouldn't be included in the build. unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for extensions ---------------------------------------------------- autoclass_content = 'init' doctest_path = [os.path.abspath('..')] doctest_test_doctest_blocks = '' doctest_global_setup = """ from pymongo.mongo_client import MongoClient client = MongoClient() client.drop_database("doctest_test") db = client.doctest_test """ # -- Options for HTML output --------------------------------------------------- # Theme gratefully vendored from CPython source. html_theme = "pydoctheme" html_theme_path = ["."] html_theme_options = { 'collapsiblesidebar': True, 'googletag': False } # Additional static files. html_static_path = ['static'] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'PyMongo' + release.replace('.', '_') # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'PyMongo.tex', u'PyMongo Documentation', u'Michael Dirolf', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_use_modindex = True intersphinx_mapping = { 'gevent': ('http://www.gevent.org/', None), } pymongo-3.6.1/doc/migrate-to-pymongo3.rst0000644000076600000240000003711613245617773020602 0ustar shanestaff00000000000000PyMongo 3 Migration Guide ========================= .. contents:: .. testsetup:: from pymongo import MongoClient, ReadPreference client = MongoClient() collection = client.my_database.my_collection PyMongo 3 is a partial rewrite bringing a large number of improvements. It also brings a number of backward breaking changes. This guide provides a roadmap for migrating an existing application from PyMongo 2.x to 3.x or writing libraries that will work with both PyMongo 2.x and 3.x. PyMongo 2.9 ----------- The first step in any successful migration involves upgrading to, or requiring, at least PyMongo 2.9. If your project has a requirements.txt file, add the line "pymongo >= 2.9, < 3.0" until you have completely migrated to PyMongo 3. Most of the key new methods and options from PyMongo 3.0 are backported in PyMongo 2.9 making migration much easier. Enable Deprecation Warnings --------------------------- Starting with PyMongo 2.9, :exc:`DeprecationWarning` is raised by most methods removed in PyMongo 3.0. Make sure you enable runtime warnings to see where deprecated functions and methods are being used in your application:: python -Wd Warnings can also be changed to errors:: python -Wd -Werror .. note:: Not all deprecated features raise :exc:`DeprecationWarning` when used. For example, the :meth:`~pymongo.collection.Collection.find` options renamed in PyMongo 3.0 do not raise :exc:`DeprecationWarning` when used in PyMongo 2.x. See also `Removed features with no migration path`_. CRUD API -------- Changes to find() and find_one() ................................ "spec" renamed "filter" ~~~~~~~~~~~~~~~~~~~~~~~ The `spec` option has been renamed to `filter`. Code like this:: >>> cursor = collection.find(spec={"a": 1}) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find(filter={"a": 1}) or this with any version of PyMongo: .. doctest:: >>> cursor = collection.find({"a": 1}) "fields" renamed "projection" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `fields` option has been renamed to `projection`. Code like this:: >>> cursor = collection.find({"a": 1}, fields={"_id": False}) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find({"a": 1}, projection={"_id": False}) or this with any version of PyMongo: .. doctest:: >>> cursor = collection.find({"a": 1}, {"_id": False}) "partial" renamed "allow_partial_results" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `partial` option has been renamed to `allow_partial_results`. Code like this:: >>> cursor = collection.find({"a": 1}, partial=True) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find({"a": 1}, allow_partial_results=True) "timeout" replaced by "no_cursor_timeout" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `timeout` option has been replaced by `no_cursor_timeout`. Code like this:: >>> cursor = collection.find({"a": 1}, timeout=False) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find({"a": 1}, no_cursor_timeout=True) "network_timeout" is removed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `network_timeout` option has been removed. This option was always the wrong solution for timing out long running queries and should never be used in production. Starting with **MongoDB 2.6** you can use the $maxTimeMS query modifier. Code like this:: # Set a 5 second select() timeout. >>> cursor = collection.find({"a": 1}, network_timeout=5) can be changed to this with PyMongo 2.9 or later: .. doctest:: # Set a 5 second (5000 millisecond) server side query timeout. >>> cursor = collection.find({"a": 1}, modifiers={"$maxTimeMS": 5000}) or with PyMongo 3.5 or later: >>> cursor = collection.find({"a": 1}, max_time_ms=5000) or with any version of PyMongo: .. doctest:: >>> cursor = collection.find({"$query": {"a": 1}, "$maxTimeMS": 5000}) .. seealso:: `$maxTimeMS `_ Tailable cursors ~~~~~~~~~~~~~~~~ The `tailable` and `await_data` options have been replaced by `cursor_type`. Code like this:: >>> cursor = collection.find({"a": 1}, tailable=True) >>> cursor = collection.find({"a": 1}, tailable=True, await_data=True) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo import CursorType >>> cursor = collection.find({"a": 1}, cursor_type=CursorType.TAILABLE) >>> cursor = collection.find({"a": 1}, cursor_type=CursorType.TAILABLE_AWAIT) Other removed options ~~~~~~~~~~~~~~~~~~~~~ The `slave_okay`, `read_preference`, `tag_sets`, and `secondary_acceptable_latency_ms` options have been removed. See the `Read Preferences`_ section for solutions. The aggregate method always returns a cursor ............................................ PyMongo 2.6 added an option to return an iterable cursor from :meth:`~pymongo.collection.Collection.aggregate`. In PyMongo 3 :meth:`~pymongo.collection.Collection.aggregate` always returns a cursor. Use the `cursor` option for consistent behavior with PyMongo 2.9 and later: .. doctest:: >>> for result in collection.aggregate([], cursor={}): ... pass Read Preferences ---------------- The "slave_okay" option is removed .................................. The `slave_okay` option is removed from PyMongo's API. The secondaryPreferred read preference provides the same behavior. Code like this:: >>> client = MongoClient(slave_okay=True) can be changed to this with PyMongo 2.9 or newer: .. doctest:: >>> client = MongoClient(readPreference="secondaryPreferred") The "read_preference" attribute is immutable ............................................ Code like this:: >>> from pymongo import ReadPreference >>> db = client.my_database >>> db.read_preference = ReadPreference.SECONDARY can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> db = client.get_database("my_database", ... read_preference=ReadPreference.SECONDARY) Code like this:: >>> cursor = collection.find({"a": 1}, ... read_preference=ReadPreference.SECONDARY) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> coll2 = collection.with_options(read_preference=ReadPreference.SECONDARY) >>> cursor = coll2.find({"a": 1}) .. seealso:: :meth:`~pymongo.database.Database.get_collection` The "tag_sets" option and attribute are removed ............................................... The `tag_sets` MongoClient option is removed. The `read_preference` option can be used instead. Code like this:: >>> client = MongoClient( ... read_preference=ReadPreference.SECONDARY, ... tag_sets=[{"dc": "ny"}, {"dc": "sf"}]) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo.read_preferences import Secondary >>> client = MongoClient(read_preference=Secondary([{"dc": "ny"}])) To change the tags sets for a Database or Collection, code like this:: >>> db = client.my_database >>> db.read_preference = ReadPreference.SECONDARY >>> db.tag_sets = [{"dc": "ny"}] can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> db = client.get_database("my_database", ... read_preference=Secondary([{"dc": "ny"}])) Code like this:: >>> cursor = collection.find( ... {"a": 1}, ... read_preference=ReadPreference.SECONDARY, ... tag_sets=[{"dc": "ny"}]) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo.read_preferences import Secondary >>> coll2 = collection.with_options( ... read_preference=Secondary([{"dc": "ny"}])) >>> cursor = coll2.find({"a": 1}) .. seealso:: :meth:`~pymongo.database.Database.get_collection` The "secondary_acceptable_latency_ms" option and attribute are removed ...................................................................... PyMongo 2.x supports `secondary_acceptable_latency_ms` as an option to methods throughout the driver, but mongos only supports a global latency option. PyMongo 3.x has changed to match the behavior of mongos, allowing migration from a single server, to a replica set, to a sharded cluster without a surprising change in server selection behavior. A new option, `localThresholdMS`, is available through MongoClient and should be used in place of `secondaryAcceptableLatencyMS`. Code like this:: >>> client = MongoClient(readPreference="nearest", ... secondaryAcceptableLatencyMS=100) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> client = MongoClient(readPreference="nearest", ... localThresholdMS=100) Write Concern ------------- The "safe" option is removed ............................ In PyMongo 3 the `safe` option is removed from the entire API. :class:`~pymongo.mongo_client.MongoClient` has always defaulted to acknowledged write operations and continues to do so in PyMongo 3. The "write_concern" attribute is immutable .......................................... The `write_concern` attribute is immutable in PyMongo 3. Code like this:: >>> client = MongoClient() >>> client.write_concern = {"w": "majority"} can be changed to this with any version of PyMongo: .. doctest:: >>> client = MongoClient(w="majority") Code like this:: >>> db = client.my_database >>> db.write_concern = {"w": "majority"} can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo import WriteConcern >>> db = client.get_database("my_database", ... write_concern=WriteConcern(w="majority")) The new CRUD API write methods do not accept write concern options. Code like this:: >>> oid = collection.insert({"a": 2}, w="majority") can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo import WriteConcern >>> coll2 = collection.with_options( ... write_concern=WriteConcern(w="majority")) >>> oid = coll2.insert({"a": 2}) .. seealso:: :meth:`~pymongo.database.Database.get_collection` Codec Options ------------- The "document_class" attribute is removed ......................................... Code like this:: >>> from bson.son import SON >>> client = MongoClient() >>> client.document_class = SON can be replaced by this in any version of PyMongo: .. doctest:: >>> from bson.son import SON >>> client = MongoClient(document_class=SON) or to change the `document_class` for a :class:`~pymongo.database.Database` with PyMongo 2.9 or later: .. doctest:: >>> from bson.codec_options import CodecOptions >>> from bson.son import SON >>> db = client.get_database("my_database", CodecOptions(SON)) .. seealso:: :meth:`~pymongo.database.Database.get_collection` and :meth:`~pymongo.collection.Collection.with_options` The "uuid_subtype" option and attribute are removed ................................................... Code like this:: >>> from bson.binary import JAVA_LEGACY >>> db = client.my_database >>> db.uuid_subtype = JAVA_LEGACY can be replaced by this with PyMongo 2.9 or later: .. doctest:: >>> from bson.binary import JAVA_LEGACY >>> from bson.codec_options import CodecOptions >>> db = client.get_database("my_database", ... CodecOptions(uuid_representation=JAVA_LEGACY)) .. seealso:: :meth:`~pymongo.database.Database.get_collection` and :meth:`~pymongo.collection.Collection.with_options` MongoClient ----------- MongoClient connects asynchronously ................................... In PyMongo 3, the :class:`~pymongo.mongo_client.MongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :exc:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor :exc:`~pymongo.errors.ConfigurationError` if the user’s credentials are wrong. Instead, the constructor returns immediately and launches the connection process on background threads. The `connect` option is added to control whether these threads are started immediately, or when the client is first used. For consistent behavior in PyMongo 2.x and PyMongo 3.x, code like this:: >>> from pymongo.errors import ConnectionFailure >>> try: ... client = MongoClient() ... except ConnectionFailure: ... print("Server not available") >>> can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo.errors import ConnectionFailure >>> client = MongoClient(connect=False) >>> try: ... result = client.admin.command("ismaster") ... except ConnectionFailure: ... print("Server not available") >>> Any operation can be used to determine if the server is available. We choose the "ismaster" command here because it is cheap and does not require auth, so it is a simple way to check whether the server is available. The max_pool_size parameter is removed ...................................... PyMongo 3 replaced the max_pool_size parameter with support for the MongoDB URI `maxPoolSize` option. Code like this:: >>> client = MongoClient(max_pool_size=10) can be replaced by this with PyMongo 2.9 or later: .. doctest:: >>> client = MongoClient(maxPoolSize=10) >>> client = MongoClient("mongodb://localhost:27017/?maxPoolSize=10") The "disconnect" method is removed .................................. Code like this:: >>> client.disconnect() can be replaced by this with PyMongo 2.9 or later: .. doctest:: >>> client.close() The host and port attributes are removed ........................................ Code like this:: >>> host = client.host >>> port = client.port can be replaced by this with PyMongo 2.9 or later: .. doctest:: >>> address = client.address >>> host, port = address or (None, None) BSON ---- "as_class", "tz_aware", and "uuid_subtype" are removed ...................................................... The `as_class`, `tz_aware`, and `uuid_subtype` parameters have been removed from the functions provided in :mod:`bson`. Code like this:: >>> from bson import BSON >>> from bson.son import SON >>> encoded = BSON.encode({"a": 1}, as_class=SON) can be replaced by this in PyMongo 2.9 or later: .. doctest:: >>> from bson import BSON >>> from bson.codec_options import CodecOptions >>> from bson.son import SON >>> encoded = BSON.encode({"a": 1}, codec_options=CodecOptions(SON)) Removed features with no migration path --------------------------------------- MasterSlaveConnection is removed ................................ Master slave deployments are deprecated in MongoDB. Starting with MongoDB 3.0 a replica set can have up to 50 members and that limit is likely to be removed in later releases. We recommend migrating to replica sets instead. Requests are removed .................... The client methods `start_request`, `in_request`, and `end_request` are removed. Requests were designed to make read-your-writes consistency more likely with the w=0 write concern. Additionally, a thread in a request used the same member for all secondary reads in a replica set. To ensure read-your-writes consistency in PyMongo 3.0, do not override the default write concern with w=0, and do not override the default read preference of PRIMARY. The "compile_re" option is removed .................................. In PyMongo 3 regular expressions are never compiled to Python match objects. The "use_greenlets" option is removed ..................................... The `use_greenlets` option was meant to allow use of PyMongo with Gevent without the use of gevent.monkey.patch_threads(). This option caused a lot of confusion and made it difficult to support alternative asyncio libraries like Eventlet. Users of Gevent should use gevent.monkey.patch_all() instead. .. seealso:: :doc:`examples/gevent` pymongo-3.6.1/doc/tools.rst0000644000076600000240000002050613245621354016102 0ustar shanestaff00000000000000Tools ===== Many tools have been written for working with **PyMongo**. If you know of or have created a tool for working with MongoDB from Python please list it here. .. note:: We try to keep this list current. As such, projects that have not been updated recently or appear to be unmaintained will occasionally be removed from the list or moved to the back (to keep the list from becoming too intimidating). If a project gets removed that is still being developed or is in active use please let us know or add it back. ORM-like Layers --------------- Some people have found that they prefer to work with a layer that has more features than PyMongo provides. Often, things like models and validation are desired. To that end, several different ORM-like layers have been written by various authors. It is our recommendation that new users begin by working directly with PyMongo, as described in the rest of this documentation. Many people have found that the features of PyMongo are enough for their needs. Even if you eventually come to the decision to use one of these layers, the time spent working directly with the driver will have increased your understanding of how MongoDB actually works. PyMODM `PyMODM `_ is an ORM-like framework on top of PyMongo. PyMODM is maintained by engineers at MongoDB, Inc. and is quick to adopt new MongoDB features. PyMODM is a "core" ODM, meaning that it provides simple, extensible functionality that can be leveraged by other libraries to target platforms like Django. At the same time, PyMODM is powerful enough to be used for developing applications on its own. Complete documentation is available on `readthedocs `_ in addition to a `Gitter channel `_ for discussing the project. Humongolus `Humongolus `_ is a lightweight ORM framework for Python and MongoDB. The name comes from the combination of MongoDB and `Homunculus `_ (the concept of a miniature though fully formed human body). Humongolus allows you to create models/schemas with robust validation. It attempts to be as pythonic as possible and exposes the pymongo cursor objects whenever possible. The code is available for download `at github `_. Tutorials and usage examples are also available at GitHub. Ming `Ming `_ (the Merciless) is a library that allows you to enforce schemas on a MongoDB database in your Python application. It was developed by `SourceForge `_ in the course of their migration to MongoDB. See the `introductory blog post `_ for more details. MongoEngine `MongoEngine `_ is another ORM-like layer on top of PyMongo. It allows you to define schemas for documents and query collections using syntax inspired by the Django ORM. The code is available on `github `_; for more information, see the `tutorial `_. MotorEngine `MotorEngine `_ is a port of MongoEngine to Motor, for asynchronous access with Tornado. It implements the same modeling APIs to be data-portable, meaning that a model defined in MongoEngine can be read in MotorEngine. The source is `available on github `_. uMongo `uMongo `_ is a Python MongoDB ODM. Its inception comes from two needs: the lack of async ODM and the difficulty to do document (un)serialization with existing ODMs. Works with multiple drivers: PyMongo, TxMongo, motor_asyncio, and mongomock. The source `is available on github `_ No longer maintained """""""""""""""""""" MongoKit The `MongoKit `_ framework is an ORM-like layer on top of PyMongo. There is also a MongoKit `google group `_. MongoAlchemy `MongoAlchemy `_ is another ORM-like layer on top of PyMongo. Its API is inspired by `SQLAlchemy `_. The code is available `on github `_; for more information, see `the tutorial `_. Minimongo `minimongo `_ is a lightweight, pythonic interface to MongoDB. It retains pymongo's query and update API, and provides a number of additional features, including a simple document-oriented interface, connection pooling, index management, and collection & database naming helpers. The `source is on github `_. Manga `Manga `_ aims to be a simpler ORM-like layer on top of PyMongo. The syntax for defining schema is inspired by the Django ORM, but Pymongo's query language is maintained. The source `is on github `_. Framework Tools --------------- This section lists tools and adapters that have been designed to work with various Python frameworks and libraries. * `Djongo `_ is a connector for using Django with MongoDB as the database backend. Use the Django Admin GUI to add and modify documents in MongoDB. The `Djongo Source Code `_ is hosted on github and the `Djongo package `_ is on pypi. * `Django MongoDB Engine `_ is a MongoDB database backend for Django that completely integrates with its ORM. For more information `see the tutorial `_. * `mango `_ provides MongoDB backends for Django sessions and authentication (bypassing :mod:`django.db` entirely). * `Django MongoEngine `_ is a MongoDB backend for Django, an `example: `_. For more information ``_ * `mongodb_beaker `_ is a project to enable using MongoDB as a backend for `beaker's `_ caching / session system. `The source is on github `_. * `Log4Mongo `_ is a flexible Python logging handler that can store logs in MongoDB using normal and capped collections. * `MongoLog `_ is a Python logging handler that stores logs in MongoDB using a capped collection. * `c5t `_ is a content-management system using TurboGears and MongoDB. * `rod.recipe.mongodb `_ is a ZC Buildout recipe for downloading and installing MongoDB. * `repoze-what-plugins-mongodb `_ is a project working to support a plugin for using MongoDB as a backend for :mod:`repoze.what`. * `mongobox `_ is a tool to run a sandboxed MongoDB instance from within a python app. * `Flask-MongoAlchemy `_ Add Flask support for MongoDB using MongoAlchemy. * `Flask-MongoKit `_ Flask extension to better integrate MongoKit into Flask. * `Flask-PyMongo `_ Flask-PyMongo bridges Flask and PyMongo. Alternative Drivers ------------------- These are alternatives to PyMongo. * `Motor `_ is a full-featured, non-blocking MongoDB driver for Python Tornado applications. * `TxMongo `_ is an asynchronous Twisted Python driver for MongoDB. * `MongoMock `_ is a small library to help testing Python code that interacts with MongoDB via Pymongo. pymongo-3.6.1/doc/tutorial.rst0000644000076600000240000003265613245617773016630 0ustar shanestaff00000000000000Tutorial ======== .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('test-database') This tutorial is intended as an introduction to working with **MongoDB** and **PyMongo**. Prerequisites ------------- Before we start, make sure that you have the **PyMongo** distribution :doc:`installed `. In the Python shell, the following should run without raising an exception: .. doctest:: >>> import pymongo This tutorial also assumes that a MongoDB instance is running on the default host and port. Assuming you have `downloaded and installed `_ MongoDB, you can start it like so: .. code-block:: bash $ mongod Making a Connection with MongoClient ------------------------------------ The first step when working with **PyMongo** is to create a :class:`~pymongo.mongo_client.MongoClient` to the running **mongod** instance. Doing so is easy: .. doctest:: >>> from pymongo import MongoClient >>> client = MongoClient() The above code will connect on the default host and port. We can also specify the host and port explicitly, as follows: .. doctest:: >>> client = MongoClient('localhost', 27017) Or use the MongoDB URI format: .. doctest:: >>> client = MongoClient('mongodb://localhost:27017/') Getting a Database ------------------ A single instance of MongoDB can support multiple independent `databases `_. When working with PyMongo you access databases using attribute style access on :class:`~pymongo.mongo_client.MongoClient` instances: .. doctest:: >>> db = client.test_database If your database name is such that using attribute style access won't work (like ``test-database``), you can use dictionary style access instead: .. doctest:: >>> db = client['test-database'] Getting a Collection -------------------- A `collection `_ is a group of documents stored in MongoDB, and can be thought of as roughly the equivalent of a table in a relational database. Getting a collection in PyMongo works the same as getting a database: .. doctest:: >>> collection = db.test_collection or (using dictionary style access): .. doctest:: >>> collection = db['test-collection'] An important note about collections (and databases) in MongoDB is that they are created lazily - none of the above commands have actually performed any operations on the MongoDB server. Collections and databases are created when the first document is inserted into them. Documents --------- Data in MongoDB is represented (and stored) using JSON-style documents. In PyMongo we use dictionaries to represent documents. As an example, the following dictionary might be used to represent a blog post: .. doctest:: >>> import datetime >>> post = {"author": "Mike", ... "text": "My first blog post!", ... "tags": ["mongodb", "python", "pymongo"], ... "date": datetime.datetime.utcnow()} Note that documents can contain native Python types (like :class:`datetime.datetime` instances) which will be automatically converted to and from the appropriate `BSON `_ types. .. todo:: link to table of Python <-> BSON types Inserting a Document -------------------- To insert a document into a collection we can use the :meth:`~pymongo.collection.Collection.insert_one` method: .. doctest:: >>> posts = db.posts >>> post_id = posts.insert_one(post).inserted_id >>> post_id ObjectId('...') When a document is inserted a special key, ``"_id"``, is automatically added if the document doesn't already contain an ``"_id"`` key. The value of ``"_id"`` must be unique across the collection. :meth:`~pymongo.collection.Collection.insert_one` returns an instance of :class:`~pymongo.results.InsertOneResult`. For more information on ``"_id"``, see the `documentation on _id `_. After inserting the first document, the *posts* collection has actually been created on the server. We can verify this by listing all of the collections in our database: .. doctest:: >>> db.collection_names(include_system_collections=False) [u'posts'] Getting a Single Document With :meth:`~pymongo.collection.Collection.find_one` ------------------------------------------------------------------------------ The most basic type of query that can be performed in MongoDB is :meth:`~pymongo.collection.Collection.find_one`. This method returns a single document matching a query (or ``None`` if there are no matches). It is useful when you know there is only one matching document, or are only interested in the first match. Here we use :meth:`~pymongo.collection.Collection.find_one` to get the first document from the posts collection: .. doctest:: >>> import pprint >>> pprint.pprint(posts.find_one()) {u'_id': ObjectId('...'), u'author': u'Mike', u'date': datetime.datetime(...), u'tags': [u'mongodb', u'python', u'pymongo'], u'text': u'My first blog post!'} The result is a dictionary matching the one that we inserted previously. .. note:: The returned document contains an ``"_id"``, which was automatically added on insert. :meth:`~pymongo.collection.Collection.find_one` also supports querying on specific elements that the resulting document must match. To limit our results to a document with author "Mike" we do: .. doctest:: >>> pprint.pprint(posts.find_one({"author": "Mike"})) {u'_id': ObjectId('...'), u'author': u'Mike', u'date': datetime.datetime(...), u'tags': [u'mongodb', u'python', u'pymongo'], u'text': u'My first blog post!'} If we try with a different author, like "Eliot", we'll get no result: .. doctest:: >>> posts.find_one({"author": "Eliot"}) >>> .. _querying-by-objectid: Querying By ObjectId -------------------- We can also find a post by its ``_id``, which in our example is an ObjectId: .. doctest:: >>> post_id ObjectId(...) >>> pprint.pprint(posts.find_one({"_id": post_id})) {u'_id': ObjectId('...'), u'author': u'Mike', u'date': datetime.datetime(...), u'tags': [u'mongodb', u'python', u'pymongo'], u'text': u'My first blog post!'} Note that an ObjectId is not the same as its string representation: .. doctest:: >>> post_id_as_str = str(post_id) >>> posts.find_one({"_id": post_id_as_str}) # No result >>> A common task in web applications is to get an ObjectId from the request URL and find the matching document. It's necessary in this case to **convert the ObjectId from a string** before passing it to ``find_one``:: from bson.objectid import ObjectId # The web framework gets post_id from the URL and passes it as a string def get(post_id): # Convert from string to ObjectId: document = client.db.collection.find_one({'_id': ObjectId(post_id)}) .. seealso:: :ref:`web-application-querying-by-objectid` A Note On Unicode Strings ------------------------- You probably noticed that the regular Python strings we stored earlier look different when retrieved from the server (e.g. u'Mike' instead of 'Mike'). A short explanation is in order. MongoDB stores data in `BSON format `_. BSON strings are UTF-8 encoded so PyMongo must ensure that any strings it stores contain only valid UTF-8 data. Regular strings () are validated and stored unaltered. Unicode strings () are encoded UTF-8 first. The reason our example string is represented in the Python shell as u'Mike' instead of 'Mike' is that PyMongo decodes each BSON string to a Python unicode string, not a regular str. `You can read more about Python unicode strings here `_. Bulk Inserts ------------ In order to make querying a little more interesting, let's insert a few more documents. In addition to inserting a single document, we can also perform *bulk insert* operations, by passing a list as the first argument to :meth:`~pymongo.collection.Collection.insert_many`. This will insert each document in the list, sending only a single command to the server: .. doctest:: >>> new_posts = [{"author": "Mike", ... "text": "Another post!", ... "tags": ["bulk", "insert"], ... "date": datetime.datetime(2009, 11, 12, 11, 14)}, ... {"author": "Eliot", ... "title": "MongoDB is fun", ... "text": "and pretty easy too!", ... "date": datetime.datetime(2009, 11, 10, 10, 45)}] >>> result = posts.insert_many(new_posts) >>> result.inserted_ids [ObjectId('...'), ObjectId('...')] There are a couple of interesting things to note about this example: - The result from :meth:`~pymongo.collection.Collection.insert_many` now returns two :class:`~bson.objectid.ObjectId` instances, one for each inserted document. - ``new_posts[1]`` has a different "shape" than the other posts - there is no ``"tags"`` field and we've added a new field, ``"title"``. This is what we mean when we say that MongoDB is *schema-free*. Querying for More Than One Document ----------------------------------- To get more than a single document as the result of a query we use the :meth:`~pymongo.collection.Collection.find` method. :meth:`~pymongo.collection.Collection.find` returns a :class:`~pymongo.cursor.Cursor` instance, which allows us to iterate over all matching documents. For example, we can iterate over every document in the ``posts`` collection: .. doctest:: >>> for post in posts.find(): ... pprint.pprint(post) ... {u'_id': ObjectId('...'), u'author': u'Mike', u'date': datetime.datetime(...), u'tags': [u'mongodb', u'python', u'pymongo'], u'text': u'My first blog post!'} {u'_id': ObjectId('...'), u'author': u'Mike', u'date': datetime.datetime(...), u'tags': [u'bulk', u'insert'], u'text': u'Another post!'} {u'_id': ObjectId('...'), u'author': u'Eliot', u'date': datetime.datetime(...), u'text': u'and pretty easy too!', u'title': u'MongoDB is fun'} Just like we did with :meth:`~pymongo.collection.Collection.find_one`, we can pass a document to :meth:`~pymongo.collection.Collection.find` to limit the returned results. Here, we get only those documents whose author is "Mike": .. doctest:: >>> for post in posts.find({"author": "Mike"}): ... pprint.pprint(post) ... {u'_id': ObjectId('...'), u'author': u'Mike', u'date': datetime.datetime(...), u'tags': [u'mongodb', u'python', u'pymongo'], u'text': u'My first blog post!'} {u'_id': ObjectId('...'), u'author': u'Mike', u'date': datetime.datetime(...), u'tags': [u'bulk', u'insert'], u'text': u'Another post!'} Counting -------- If we just want to know how many documents match a query we can perform a :meth:`~pymongo.cursor.Cursor.count` operation instead of a full query. We can get a count of all of the documents in a collection: .. doctest:: >>> posts.count() 3 or just of those documents that match a specific query: .. doctest:: >>> posts.find({"author": "Mike"}).count() 2 Range Queries ------------- MongoDB supports many different types of `advanced queries `_. As an example, lets perform a query where we limit results to posts older than a certain date, but also sort the results by author: .. doctest:: >>> d = datetime.datetime(2009, 11, 12, 12) >>> for post in posts.find({"date": {"$lt": d}}).sort("author"): ... pprint.pprint(post) ... {u'_id': ObjectId('...'), u'author': u'Eliot', u'date': datetime.datetime(...), u'text': u'and pretty easy too!', u'title': u'MongoDB is fun'} {u'_id': ObjectId('...'), u'author': u'Mike', u'date': datetime.datetime(...), u'tags': [u'bulk', u'insert'], u'text': u'Another post!'} Here we use the special ``"$lt"`` operator to do a range query, and also call :meth:`~pymongo.cursor.Cursor.sort` to sort the results by author. Indexing -------- Adding indexes can help accelerate certain queries and can also add additional functionality to querying and storing documents. In this example, we'll demonstrate how to create a `unique index `_ on a key that rejects documents whose value for that key already exists in the index. First, we'll need to create the index: .. doctest:: >>> result = db.profiles.create_index([('user_id', pymongo.ASCENDING)], ... unique=True) >>> sorted(list(db.profiles.index_information())) [u'_id_', u'user_id_1'] Notice that we have two indexes now: one is the index on ``_id`` that MongoDB creates automatically, and the other is the index on ``user_id`` we just created. Now let's set up some user profiles: .. doctest:: >>> user_profiles = [ ... {'user_id': 211, 'name': 'Luke'}, ... {'user_id': 212, 'name': 'Ziltoid'}] >>> result = db.profiles.insert_many(user_profiles) The index prevents us from inserting a document whose ``user_id`` is already in the collection: .. doctest:: :options: +IGNORE_EXCEPTION_DETAIL >>> new_profile = {'user_id': 213, 'name': 'Drew'} >>> duplicate_profile = {'user_id': 212, 'name': 'Tommy'} >>> result = db.profiles.insert_one(new_profile) # This is fine. >>> result = db.profiles.insert_one(duplicate_profile) Traceback (most recent call last): DuplicateKeyError: E11000 duplicate key error index: test_database.profiles.$user_id_1 dup key: { : 212 } .. seealso:: The MongoDB documentation on `indexes `_ pymongo-3.6.1/doc/__init__.py0000644000076600000240000000000013245621354016304 0ustar shanestaff00000000000000pymongo-3.6.1/doc/mongo_extensions.py0000644000076600000240000000566613245621354020172 0ustar shanestaff00000000000000# Copyright 2009-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """MongoDB specific extensions to Sphinx.""" from docutils import nodes from docutils.parsers import rst from sphinx import addnodes class mongodoc(nodes.Admonition, nodes.Element): pass class mongoref(nodes.reference): pass def visit_mongodoc_node(self, node): self.visit_admonition(node, "seealso") def depart_mongodoc_node(self, node): self.depart_admonition(node) def visit_mongoref_node(self, node): atts = {"class": "reference external", "href": node["refuri"], "name": node["name"]} self.body.append(self.starttag(node, 'a', '', **atts)) def depart_mongoref_node(self, node): self.body.append('') if not isinstance(node.parent, nodes.TextElement): self.body.append('\n') class MongodocDirective(rst.Directive): has_content = True required_arguments = 0 optional_arguments = 0 final_argument_whitespace = False option_spec = {} def run(self): node = mongodoc() title = 'The MongoDB documentation on' node += nodes.title(title, title) self.state.nested_parse(self.content, self.content_offset, node) return [node] def process_mongodoc_nodes(app, doctree, fromdocname): for node in doctree.traverse(mongodoc): anchor = None for name in node.parent.parent.traverse(addnodes.desc_signature): anchor = name["ids"][0] break if not anchor: for name in node.parent.traverse(nodes.section): anchor = name["ids"][0] break for para in node.traverse(nodes.paragraph): tag = str(para.traverse()[1]) link = mongoref("", "") link["refuri"] = "http://dochub.mongodb.org/core/%s" % tag link["name"] = anchor link.append(nodes.emphasis(tag, tag)) new_para = nodes.paragraph() new_para += link node.replace(para, new_para) def setup(app): app.add_node(mongodoc, html=(visit_mongodoc_node, depart_mongodoc_node), latex=(visit_mongodoc_node, depart_mongodoc_node), text=(visit_mongodoc_node, depart_mongodoc_node)) app.add_node(mongoref, html=(visit_mongoref_node, depart_mongoref_node)) app.add_directive("mongodoc", MongodocDirective) app.connect("doctree-resolved", process_mongodoc_nodes) pymongo-3.6.1/doc/compatibility-policy.rst0000644000076600000240000000435213245621354021111 0ustar shanestaff00000000000000Compatibility Policy ==================== Semantic Versioning ------------------- PyMongo's version numbers follow `semantic versioning`_: each version number is structured "major.minor.patch". Patch releases fix bugs, minor releases add features (and may fix bugs), and major releases include API changes that break backwards compatibility (and may add features and fix bugs). Deprecation ----------- Before we remove a feature in a major release, PyMongo's maintainers make an effort to release at least one minor version that *deprecates* it. We add "**DEPRECATED**" to the feature's documentation, and update the code to raise a `DeprecationWarning`_. You can ensure your code is future-proof by running your code with the latest PyMongo release and looking for DeprecationWarnings. Starting with Python 2.7, the interpreter silences DeprecationWarnings by default. For example, the following code uses the deprecated ``insert`` method but does not raise any warning: .. code-block:: python # "insert.py" from pymongo import MongoClient client = MongoClient() client.test.test.insert({}) To print deprecation warnings to stderr, run python with "-Wd":: $ python -Wd insert.py insert.py:4: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead. client.test.test.insert({}) You can turn warnings into exceptions with "python -We":: $ python -We insert.py Traceback (most recent call last): File "insert.py", line 4, in client.test.test.insert({}) File "/home/durin/work/mongo-python-driver/pymongo/collection.py", line 2906, in insert "instead.", DeprecationWarning, stacklevel=2) DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead. If your own code's test suite passes with "python -We" then it uses no deprecated PyMongo features. .. seealso:: The Python documentation on `the warnings module`_, and `the -W command line option`_. .. _semantic versioning: http://semver.org/ .. _DeprecationWarning: https://docs.python.org/2/library/exceptions.html#exceptions.DeprecationWarning .. _the warnings module: https://docs.python.org/2/library/warnings.html .. _the -W command line option: https://docs.python.org/2/using/cmdline.html#cmdoption-W pymongo-3.6.1/doc/pydoctheme/0000755000076600000240000000000013246104133016336 5ustar shanestaff00000000000000pymongo-3.6.1/doc/pydoctheme/theme.conf0000644000076600000240000000104513245617773020331 0ustar shanestaff00000000000000[theme] inherit = default stylesheet = pydoctheme.css pygments_style = sphinx [options] bodyfont = 'Lucida Grande', Arial, sans-serif headfont = 'Lucida Grande', Arial, sans-serif footerbgcolor = white footertextcolor = #555555 relbarbgcolor = white relbartextcolor = #666666 relbarlinkcolor = #444444 sidebarbgcolor = white sidebartextcolor = #444444 sidebarlinkcolor = #444444 bgcolor = white textcolor = #222222 linkcolor = #0090c0 visitedlinkcolor = #00608f headtextcolor = #1a1a1a headbgcolor = white headlinkcolor = #aaaaaa googletag = False pymongo-3.6.1/doc/pydoctheme/static/0000755000076600000240000000000013246104133017625 5ustar shanestaff00000000000000pymongo-3.6.1/doc/pydoctheme/static/pydoctheme.css0000644000076600000240000000526713245617773022534 0ustar shanestaff00000000000000@import url("default.css"); body { background-color: white; margin-left: 1em; margin-right: 1em; } div.related { margin-bottom: 1.2em; padding: 0.5em 0; border-top: 1px solid #ccc; margin-top: 0.5em; } div.related a:hover { color: #0095C4; } div.related:first-child { border-top: 0; border-bottom: 1px solid #ccc; } div.sphinxsidebar { background-color: #eeeeee; border-radius: 5px; line-height: 130%; font-size: smaller; } div.sphinxsidebar h3, div.sphinxsidebar h4 { margin-top: 1.5em; } div.sphinxsidebarwrapper > h3:first-child { margin-top: 0.2em; } div.sphinxsidebarwrapper > ul > li > ul > li { margin-bottom: 0.4em; } div.sphinxsidebar a:hover { color: #0095C4; } div.sphinxsidebar input { font-family: 'Lucida Grande',Arial,sans-serif; border: 1px solid #999999; font-size: smaller; border-radius: 3px; } div.sphinxsidebar input[type=text] { max-width: 150px; } div.body { padding: 0 0 0 1.2em; } div.body p { line-height: 140%; } div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { margin: 0; border: 0; padding: 0.3em 0; } div.body hr { border: 0; background-color: #ccc; height: 1px; } div.body pre { border-radius: 3px; border: 1px solid #ac9; } div.body div.admonition, div.body div.impl-detail { border-radius: 3px; } div.body div.impl-detail > p { margin: 0; } div.body div.seealso { border: 1px solid #dddd66; } div.body a { color: #0072aa; } div.body a:visited { color: #6363bb; } div.body a:hover { color: #00B0E4; } tt, code, pre { font-family: monospace, sans-serif; font-size: 96.5%; } div.body tt, div.body code { border-radius: 3px; } div.body tt.descname, div.body code.descname { font-size: 120%; } div.body tt.xref, div.body a tt, div.body code.xref, div.body a code { font-weight: normal; } .deprecated { border-radius: 3px; } table.docutils { border: 1px solid #ddd; min-width: 20%; border-radius: 3px; margin-top: 10px; margin-bottom: 10px; } table.docutils td, table.docutils th { border: 1px solid #ddd !important; border-radius: 3px; } table p, table li { text-align: left !important; } table.docutils th { background-color: #eee; padding: 0.3em 0.5em; } table.docutils td { background-color: white; padding: 0.3em 0.5em; } table.footnote, table.footnote td { border: 0 !important; } div.footer { line-height: 150%; margin-top: -2em; text-align: right; width: auto; margin-right: 10px; } div.footer a:hover { color: #0095C4; } .refcount { color: #060; } .stableabi { color: #229; } pymongo-3.6.1/doc/atlas.rst0000644000076600000240000000423013245621354016042 0ustar shanestaff00000000000000Using PyMongo with MongoDB Atlas ================================ `Atlas `_ is MongoDB, Inc.'s hosted MongoDB as a service offering. To connect to Atlas, pass the connection string provided by Atlas to :class:`~pymongo.mongo_client.MongoClient`:: client = pymongo.MongoClient() Connections to Atlas require TLS/SSL. For connections using TLS/SSL, PyMongo may require third party dependencies as determined by your version of Python. With PyMongo 3.3+, you can install PyMongo 3.3+ and any TLS/SSL-related dependencies using the following pip command:: $ python -m pip install pymongo[tls] Earlier versions of PyMongo require you to manually install the dependencies. For a list of TLS/SSL-related dependencies, see :doc:`examples/tls`. .. warning:: Industry best practices, and some regulations, require the use of TLS 1.1 or newer. Though no application changes are required for PyMongo to make use of the newest protocols, some operating systems or versions may not provide an OpenSSL version new enough to support them. Users of macOS older than 10.13 (High Sierra) will need to install Python from `python.org`_, `homebrew`_, `macports`_, or another similar source. Users of Linux or other non-macOS Unix can check their OpenSSL version like this:: $ openssl version If the version number is less than 1.0.1 support for TLS 1.1 or newer is not available. Contact your operating system vendor for a solution or upgrade to a newer distribution. You can check your Python interpreter by installing the `requests`_ module and executing the following command:: python -c "import requests; print(requests.get('https://www.howsmyssl.com/a/check', verify=False).json()['tls_version'])" You should see "TLS 1.X" where X is >= 1. You can read more about TLS versions and their security implications here: ``_ .. _python.org: https://www.python.org/downloads/ .. _homebrew: https://brew.sh/ .. _macports: https://www.macports.org/ .. _requests: https://pypi.python.org/pypi/requests pymongo-3.6.1/doc/static/0000755000076600000240000000000013246104133015464 5ustar shanestaff00000000000000pymongo-3.6.1/doc/static/periodic-executor-refs.png0000644000076600000240000011524213245617773022610 0ustar shanestaff00000000000000PNG  IHDR?{LsRGB@IDATxUǯ-v'6"*݉-*⫂5.l9<ĝ{w̹;Ap31 Cxa2Q~AkZk>`@+"`ʯ{lΔ=!ВkV[ !믿/R/B?'L~ᇜ~r>dM9934eYrg?nmsL3MuUU'|}'?cg`i߾C!9ngQ\38*lEw=Q^'(O~+=ߔ{-n饗m:S9ܔ_ܺc,^z)֣X^^,nXc6[,QX?(d7|uuYjV_}\sy*) @S 0~xcGyD>M>jA@V]uU2˸駟!u&?{w+^~eF&x]Ν݆nF  GaÆ#F^{MJЩS'k馛.s~{9裏7knos j)m123]c[mͲ2kou]!=Gn]vqM5> s+ݨQt1gϞjzn)hiXla%o`}q]tqF,BԊ3r`/w_|&lk/׭[7u+9Q~wwwk=SnEqGy}SOJB1dK/uoA+0'z4Gqw}ݴN[GM0*G%\`w]ؙ|.r֕38GLϭS~eex' Z]R6*f'0up駻AW^]qnWR jv!sTS"8SMewVƱx yeW}2eݻw(vaA)D\ve裏V[o"+<0 XZ,S|n__|E7n8 {+QZU hLBM7T7 .8}k(/*[L DLUcd!㏫s27tS"露nuu]vunvٜ_YC`PJ*n-p7xcnfߪF.,rH!UK`l!Vcg 3ڠͽlG@ MǎqE<c 6bpnaVmӌcǎU9B.D\wuK}wm {Bc I/ZEF{꘍U Aidwv #嗆N6D7ȓqGr3"|H.W(Q(;˝:âSHl*CJL]Q%Li׮]p`f+$*BA>2d+ # 3<̣bO+qdطo_OzH|]reeZ.H#cu3[::1w⋫1~+[iԒfAÍ6HdwoEWiLBw[nI;CAqd\##a9[9GuL^P$(|z-z 7JKyPͯ:P; TXT7pCT2>'p9rxrzc\$AaCłҎI:T0WL C~aj?$qZwf @l2?ݻwfŚ ?OIUZ.SXd N:>=t|c=6q?eN/T}|U/kQXZlwaf 21J rcE)YIMIp/]xչsu.?s1dҲ2 0\s5sF o!L 6C^XقJKh,S2aF%ϼ5}S~hwCeƏSfc4.@,*dwYge2,~p|J9(>##\!1Wg! O[^e+:\4XeI7[XeU XH!IRkY\#Hɐ a7C !,t!YSNѕ_Cg~֥)rS#_}UQzZPZ SY=5/rDzS] A9%M_*-kĉ~Bvm7U~xaxJZIWP w9]H{;8Q[}wܱPUd0*B@A W [_IoJY z ق:rHpwk0pb%.x %!RY9TY(2+I7|sL\\/\}Ձ02Kl zhgΝI<9!y ?w=Ju֍Ylow5 <^ó(_la?'T %&M,.VWY"2˕9BHc1%[*KX k:-8t?cٻCB ٹHfe !PPF|˝ڐX~А{,^;oA+NV}ɮ2 -6'ԹC)&Sa K;ud5Rf Vs~廤4^OSWNFK/s`g]zWj3}kvbKF;f~rlt >ljJx[ + ,H8] Crh"ɝ4EBݡ~2o)+PUR0Vc!^Vvgh!dߢXM2Xtq)yփ +-&az0#M̵`KJ/xBvU:˪jB`6Se&LPvi bW,Xxv_㰶;!{bs/%^HY~H09ҍ=nk EY$,?ABJ@-,D}Pt uqw;1mҥK(k [|u` :wR6V{0[| 7du~wV% (;SfK⣭6{> "P/{KMF2ϏK)[MA'ZZz[*6fEl/-=ih9{]wyۉ!P *W؜_@i@6InS~kR/{KLӔjk vSC>vo6ɼڔ_2jDҊpڂG+9xF,t$27?2aC n5>@V@xzS Z{z͵6h{K~XoRAeo/wC e!nLQZ*4@[]tQ׾}{AŔ_{Ž(eY&\]ZcL:qroak_ҟ(!R iQ~fUv!D{K\Q~ƍs;7nms~ug#lvm,IKRoY]S";u.b7qDmk[AgyVB -(Q[x[*f^xZk)aTSMv}w7,/A(j24( CN~-q箾jMbg}ӛ6B:!r]l B\$ݺus:t!1t&#`ʯ~ C H/"gSht %!P?_~ yɠv'k@,@K (s~f83uAϮ5BDlaOt(?p*ڛԞz1&k/wie oرqV1˯bDC Z|<SL u>;Βs)nw5r@u]z뭱e[` 'Ql؛^:7>c^~mf">?ЍZiM;n9ps9kE/: 6`!(_q_=Z˛iJ+V[m5V_}QH>Ӷ"*dl (%")~aD1a V^yeK]XĊ¢ܫ^yGH{^xa׵kW3nʯ. atR+ n.]Ν;֬o' Q(DVQ;.0 iFÔ_ 4wZwPGM@#GnMS`~{dxlR*NJݐ!C[o~T\^@c[orhPX&En \IKC9DޛoK颒fh{3<%{#FBKԨkv"c9R.m馎U+BDS+Ygdí暪CMj2嗚BvmƺB;V蚤[jT b qIzª)rbO>?]Uל~]ys{Cm>0Kc f!f/d@x pgǰdlY{(D@zׇڐ/|駡٨L5 iOC N8w:&عa|O>3?P`ےcVߢ.w-k:K8_׾}{S~e'!{=n 6./F\dpuYqWW'b]l_DC\|+5N03 {/*r KIS~I1ou=ꨣܹkC6=>5ᄏ#,n,~r#XIT~FlPSP`oUWa̬nTZ+[tE5+'7ɺKv@dAF *C\^{(M3Eo9Rs0޶Og}&5a ~$ Vzj/mM5~wڻ2h2r2TX^馛z쉿KX\UvA +9dAIjF ѓ]s-;<o*{s4˯C@bqdJMz멿a*v[lM(4z)[י[k4vZ.L<#v8!8{uX0)3lذVT| 99~>loСGR@r{"R"Xȼ֪bʯU{>f {ӬnE]`laԨQ-?5`~CǼ{zz֪aV[mф܇$>GbeJcq /B_Ĕ_:̪d$яUeŧ3;2Q"Xo(̱cǺ]P$ɔWA\'ɘZ߼z5jFb85{+4A,C|PpIq,=s ^/4x펆@a|:KrNm)Fi)V!Pn!wh==vC"COV&bN2V!ᄏ^s5aiA_`l! X=ӧ8plFZ垦ZA1cȑ#S$VےkV#4v&Ӭ-A ^,[kX[Eٰ7U;ȏd:|S~+L<3 4IS~I9w 1q!0Wzv!!qv{!K.a/ڔ_  KKrYSV5_r%i+$b/ɽguo Շki4@),IHrY[w/#%[GJ:Қrː!CR5reK(^h/-qu{-`e*fs~eÆ@0zB 2@^naȠ K-l\flnРAu#C/X *n/NrDg}2,U6oX xF'9z_V^yJNS~I1! 4W_u38cyd#a@h oVr;AL -@^^{ yy`LבhUVItS~JkH!HѣGkL)V{ʬ@Ln/^ pωۑ Y~hwC axaÆ^u!L34Y#6 #@]2r|*),?ӵk׮*$!P7|]|Ūn&!?c׷r[߯CW\ѡ/Q}= @RO>2-&X~&0˯0.5wuccKMΎ@Xn܃>s}ưsG[ͭMEjD,wygQVy>c:_=47]K {g}iʔ_6H {v]tQkA^꫻`Ii/ɽgu7!=؜!0/QLWM-@Ǎ?f,>0W3v!`4 +uM1zzF1/jmjYPz~[k &ԌC^\Ғ(GD駟t Z_U~'ďBQyPqG($tM禝vZ4Lp:.~ v"= 矻^z9k;<ǿeOGڷob 7lEB -o4\[Յ7nn<|I~$[P.3nEu-XfC4!(splZldgXds9gdf!%f1N<kZkY/8￯+RJcGۈ /y xC6Dല`=y)g}/l+br٘oxJ bD/:[x =sy,@XPvcL<2V#H=t>'&H ^~eB-Қ3 +JI*d1X&c g[wuIsr=[oU+;jR/÷V[nE_L({̟gsO={GO<b7 \"/8ZKczE,6m0}x݀ZpnJL+ JpСjn"VB XXt @~0 ,:dx㍁yJ,t%X@<%ˋ]qS,e_ k*gCp 7b(6@<#QVbXʏ}恬(jd'|Q6ʎ\{;j> K,Dp12_x yQ}~6&))QI]ѩS@OhYtO?6B'E 2/EK <U(%qȢJZS2?lھm&x뭷lGrx駃 6@_{g @9'\':a,g]ӎL$}v\uU%& q/@@FvġҲFSc!I8ʏ7>g ZY c@eq?@|F'H8Yb׊J$S~鋑gCTj}OVjmVRY*Uo %$:O,!n{B +sW&"p%8Sxmʏa,2!i&}=}`זCaÆ$߫a2DÔ>'o&%5F;0'n#OLw DC5.vObG._N[:r;Վ GaGkH0xNUVVi8%n(O"%N?4$I';yi҂V"V^GI`W˴+%/wOrD,Gw9h}p dE"?E0/WaKsjudNb#*B2S;{֥ s[|mS,L=VhO!(%~^xyk00e G0G( '[yj-*Y9._t<3%f9dSIXm:[KY #s3GJ;³.N?:ԉBc|f+cp˞_=I؀|f?^hޑ,w cn11wYHHP)Vl!:@O^('*"܍YY4iQ ⋫DM*%ڙ!˦̗*-69{zW)ߑ{뎐Lw-1/ŤKe+Zm~88ꨣtQ/',pyf(4̅|uʞ+ts?~ /r2te&eEu0ioc1*r BH..֓qJ_~ńeʨDsY=eBܧsHJW"mJ/TЪui=Չ(wFM%FqhV5؝Cl (2+hm,V ٣$D6{õzaIa "A ֋?RoiXjD^QHw}kj^e#y4Cѡze o^?LUgWwVR(cqqvucJYkꪫVv<E\]tSǛH i]f4.Lvu.S&~5\aDC R=roΗЪ6`/ "F$[ $.tl~FɼW':A;@3r)V t?+Q2KNSxHɪb!nS \SB=e ϾUxc%^^6,IR˦yg%{|#g_C֯: 5$dPڮ ّ@cC)sX2_@Z-9<>e"^!F:L: 7c=V)Vs($>PDdLrV_@De %B[pU($\ԛBK :+^tNP{lQrW96ulxP򴇗g&^|@]x0|G- 󀂒}>J3>/OIhj}뮻2wi>( C"$"F'Nr+~dh27a )wϫZyF{JSw`җʏs ĄT˩ЏwR9-pA2+n$X.ME!Xkm&ȃ vD.h(X1g\!Ѩ}E@k؉?a½ixr &IC<`0Q"ftpS11@@,ePYU2o 0,HD^V) RV)/> /\U9)&FTj.s WP`b޸LH츲c驻V@׳!m.ߤ&}c|H"C&;"%܈3T\4vбKJ4| H"]~ΤؠOJEi/1eG\ |%d N %f,~nE3(7pa$ݷo_,DA &?& pj-k1ǙRGQr.C6 @e;7:atXr@IDAT51(yc6@|&fUrh:TCT C#^QߣbҚn^*B`Ű!a&6Nt#A}<(8H%qiP#u qlg$r B)yTxT/oTd;9Ox^$mV3s/Jf3Yp eyѷ l؉8C9VeOz!Qu˃ V5Jψ%}(a9zɽe _B~'/z)(k~k~4'> JnuuY:_f8 ֍O `!Vyw>Ɍ`OB}r?'P^ԍٟ&+euق敼 (#4}<.Js%5=6:CIѯyy~$yyneo\Oڃ ?[|Oɡ JQmXnW#,>Dj?VNg^sA)-|2H)=jxO&@KJ(_DO>9a+xXX$V /Y/PLLŵCwVok^u @B Ņj!, [!adxӖZZ䚚*kJ \DhbwxQf(>%_mŦĔ_L{wVW%#\LkhժpY0`.jHohī 3׈2FXH!F+)6zҷsLhժ$b }IYJ$U.~qo3٘Yd"Bs{B8l!.۟`ITj5mht3bcIh,z oLeA$ ` +- N&U\jZ`T-iT2D}L:hS~ $~P] :TGݘ*OBvATld5i~6mILzj^UrLY8=ܖ گ\*AΉ5HǓ0HV[m뺇Q9vin 6p/&ܹsEVW~꒒5X_؇~=2vXiK/./ m,|̔_)tplܸqHM@cUK`yu(VX!M]LUR OaMM_ZlǎcؔD4o?wl 2@/.pAQ|Z6C U;6lSٶ(99('\vIR~Ě[s5<̣,˦ !e*E_H<|WL2}ݎ;~mNs曻:ʝ|Gu7_9!P-5Ֆd7hm"= Isx8&3*u޽Bj<;J*B>C }(=[H&AA<6z,<#ivZ|]tќ%pa!0a}ņU>C*8.]pޔ:UrS,6۬ͩ 塟RU)dW{3'{6BDh\ΔsRX;'qT~cƌq}y)? G1L^Z-JαuYg6U)RQn^f{Ǐbb+=W[m5U`<(lEwCrDiBd@|Z!eB2˸^Z?b;%,aRE :tP:ZX)V&n4(`W^領e72;qN[o$z_.sVkxԨQp/,!+ Nq~TX(n}aQ2T&Qn6xccdiRgroRu} yP5 +b6{++oQѣGGzg=Z'S~9tMkln5,sei n'ƒyƍ6HBƷF3ר`ꫯvW]u:bCgr9lT5bwɼ {4lt0,rU(5+:3thcg] I+RyA5x'gb=U;ja$ڸN{ /3,[OL(ExCg^??VEGn&!$guk׮`ge0Wj3.sX\s{Wե Ƥ6Zk-7rHWJ+d3l'KIB?O$=L%W1gJ/pWĆndj8p#ɛo Z;jkvNsMQ?a0矯 ՗dWCx{5|dF]w]x鼕Ŕ_뭷j?BI" P{hw}wWp&Vca\day ! M*ChVft%S~=+9gA )N:餜cOc` V $nDiꖟ {|xz-;ܓ)*J:,w7Gtː1/=}yKA%6s{:,_J޺B ŧr1 C^,uLJ.E<<$0+LTU^z92UvuWMI]M bK1dIe]7ܯaMB5~vZ=p^X9l^3s]w)s $:E믿VrQuőτ;HwY !/yGg"fjh'd99 6@_Ea 8@C@C 27)$ /sfm{@^{-J`La2ȏX u $$s_ʕ#J)| D;S O>D޴S2zꩺ ~@#<2Hf7=WhQXȁ$Z dqH! N|:$aQ U+/\ 5"!02&e>K"2gVX`E "\,zle+VL0SB{afKr$YM b 2IH^ 4[Z;zK]Y LXkٴ |H^+%)z,?.K<l+r24)xƇ-rܣ:*S4M.#qpq>g}\9,R@[d.%3=ap m q: .p; A^@y-{XF_7_ԧ~3VeOa^9JRQp2/\`vr@Ob|SQʏ a9b.3is{ҒV{+۫eU؂"k9y'XY2szrX!YaHyeS4~$''wI#gu&m ](IZ]LU0T%l\~(D_s֡o߾S-BrXEc_'LN=-R,^ex*zJGH![0CO[6b0o)&w]F !qm EbWQ./DWW%+W B L Ɂ\b$izJ2ל}ف(<]':/=cdv Xd`e܋ _\D7~x-KH!)D J<NK,|{iiNjPϗ]Eer8.saGVző[Tpy' 9䐀">zoI-Qvhp+b-cǎkV:_G(\Ia^xဗY)j!Y4;j.\϶*0r'ݼ&"V' /[Q_@ H*Wq>I/NRIz30#6(61Rd?&[Hmw-hVsL/c۰Et a-,FlPmC]m ܋/Xv~O!iC/"eɔ4UV{kmРAQbW~JKB@lX\Ybq"Ŕ_ O.!S`m@ݕIx饗Uj_B ?Yяe#)fXP'G)ocAK/uD@X!?jCLUV޹8xG /w&7U:3 ~VVz0Bߒ6np׭ڮO>rOE jxw7+0R/ :ԍ5ʉӲКŰ2[+0à` 5mV_O.$AKׯ siwKgE㮽ZVꫯԚHt6Zhrș袋?fJԲBɇ {8T* S~5VEp Ӊ 8w{SVOIO2gr^fU9??<̙?lʯ]D=UHJAs=wӴB@ 6LY"i[neQk*w}w [WULUW'hJ3dFTda#r 6<"+D9O'0}ض[ngZ\L@J5ra +ٛpšՁ!/?c<DŰ!c "8C;ȎUVY 0 2Z)8v~l>* 'J"Ffr޲faFY|Z-DL0=ܨSN8J+mm Ji/ ~uFL B0?I5 ) ߄iYאo6|s# ߹+lPޣ7K.*d2:(=sQxbwzhlԤ mMBےul7n&?+$KsUTbS@$:''q-[l1'lϮ,ua *ٔ_:+̟\ ]ZGydG7o zEm1RwI$LMo䜬 BbbhbW\QOQvm`B#FE4:;197?C` !\W%vI zmFlTD aԨQa4 S~M17FabYL"kU Uqk?\&#!`/v]^ U víFWa+h @}+HvqG +W02jo~!08S|)\kRذn[oVof ELE_J6Y5\KL_G}Hd/\Cpt6S~)} +֬j 0'7Tz)`#a6)%M,Icoرc?4W0ȫ^&fo[o5tȻZG 5oR'Ej}71aȋ7LM"5"T~b~A!B,r [¨IcoƐߍ~#_~H`d/R$,oݳLՂZ̯!1o :T35vOCLUR5oVvJ0W J :믿V +7KHgibS~qUPeh {t7es5Wt7  `/݄fež\`1)IX'|]tT~Lg^8{W,?GVRfU_ݦ ’ܝ(EY$V x5Ɨu˄ Lo/$K=v/&l2ۤ F'ߔ_ IO?Q^{5cȋذ7|S~x$?(?vu C^Ĕ_n#;Vǯ[kޣ啟qA~%@`ܸqnyƞZ?_V"Y~>$ rwIE`wڵsv[lpWvGqvNS_M;U,U oxc+Zkv{nuu Dzƌb\jaoj9PQ蓑#G.GQg!(k߇ 8w]wۯ0\ght?~?5wGķW^y%3WbV+Fj q\#\< bHL F!`ʯQHG|8+?ݡZ>q;h)0lbYqV~ .wq68iy Nxށ(?|䦙f+gy;S ֍n杸^h/%=|_!xO:$w9:d+Q}' kq48#I<ꨣ_[mI];@cIŦRҫ~՚> 4(Sg/G(_p&Gk )$rqM7]!C ?ES~v2%nKfNΥI~-w}w7S޽{^{ :T}u 'l%SM5.p==2s93܆$PMWo%A$S~x$?,f)1uȏ_~Y?x G35jn_|o-2z\rIX`UTz?A۪l;< "Zs5].]tC!T"X}Ϥ-D_guq\ÇW~'|Rk:w6xc*7$~ꩧWnA2ݻww={t믿~Ji B`)B$pC9J1F5+=C:Ƕ[2tMc"[o=ᄆ̋Ҩ>M}joJzfXp袋taa7w7k*skK-;`(Bb*סCKs=ŚRߌEX8V{1;$GB ĿkVz}饗RԴv3La+ffɉ0`ÂJ ;+AX$S~c5j;zh&8q` r>g}w}ל^Z^^<&C_`6b{ 78p@F\s5WR)ɜ8wǺ6ȑ<$YKV-3âO>sr3Eh}'_|Qux!'EL )¸$n/a[PG!'Nt]vu$""./;vT6L{(2~i]믿j>YGJz8*=1-)A,f@(t . :ûA%73Zc̔_Jaa !so;L#pg'T])(?HL #`ʯ0.q^_As}Uhn%J#oʯ4>9wkT:q?k )}f IQvXC`?9S6p5!ml+)$(sLa)Ĵ?neD\ު1ͨ)Ҩ+Ob~Ǻa\ݵkͬ{ou7Na*|Vɉn,nkƭqO 2IBeH-*@x.T#FZh@(Kcwq'*W\0ńZV7K;l-ðp 8nVQDBT꫇nvI25׆=?S]LPsE.$ 1ci CMCltJBo/ԉW[6$7Zsy$ C$wygv&s[m2$exR<~pru^\Cl+MP^>CvErW$F=mhqĺk$ ](wZ/9,08f2Yh^lQeh@U;mYϓU'ˁ _IY:_ ^ (59QEH\%u ԉr1ۜ_qlw֕?zCa8KW",!Nj DVZi%u]9ʪ8/a3\b-i쳵밆J.dS|fS~觪j U]XdrX|2#,;:ĥ=D|1!W\qsDmtf/E雂ՃP0E2t"J)|aEaHjտDn(mZ}Ck/=X(?[':Db-tU8{ŭ-mݦ){JOX8kt\sMD€뮻*#GLyMEqoe/Cθ ɼY\+*c$ UiP$I-L`vA}!ɔ_`Ʋ(oר x( UǏXy裏,mĢK+|%II+y&]vE 801 fKY`nIk I5H7KN_TSV,QsOM"Sg6dܝ_AL%=;X%mZħdkbԊ)ZKu }%. $E-?S~E! `ʯ~ 34R/%-+?7S~þawB"9ܒ.6Mzƣעo߾.2lذ gިNwݿ-UK/Kl؛GMţR~)i$ inbhCfwI^S~˲-㏜'[P7UxEkF^Fp8M{zғOT$~lț?7Ncs0W lIP~ؑB|͗{+)Ф 'zw&Q3s_}Ƥ2LUSz/KDao^/uY~m)ǔ_1dRcgu;5PҚj޶=Mi{D_AXҿs7WJ#<2qao.3-._)tR~|XnmbZk_ۮZguLS~%IArX{n-T߹$&!`Jwc$e]ٰ71]늚uD_EYqJu=|ĆIє_(RLذ7 :E^;rtM~"h׃)?{m1i~8EYlINL%"%\Igao{'9u3嗜-;t{#_7ao.r#FpiE_YZ;̭ n=pO,on뮻ܽޛ+) SL12dE8 {s{eرniE_YZ[ѣc {s/0 IE;%X‘608Cj@*0W^-sSN~o;cnNr`pa$L*fʯRZeY}27adȋILUTw]޽{9(؜ߤn`,v7 JUHy^{ޓk(w}{5gy;j*@5j8qbfv&;vt rLSTd2aTw݊9R9H뮻6 ,ρ7~dEdAfrIaϦ"0܆M?97֭R_׽m*#zws0? Ňճ;{+0b^OsIzJ+=z_UBr!K.Ϯnn/xDbbT)jjs۵knVG}Os饗*2\sM>ܓT|2ܗ!P Au$ž-ܢNЏ?x&x񎅈{EiXo¾kNM0!G)aa-l*˯W^>C$JcSN_|Sa$n ]Ȗo\8)LT $h& j R4|%(41)``"!CE %J@Nsνwfcc;wfc?{fݽZPDH֭*bK$82dDwy~ K:QwUŦ<܏ S) U~ᖺ[m}W-%]&n|[RyIxhѢEfIJYP7,MJ@ 믿.3-jVXafWjzQaqUQ.6olQ) U~ʧ7n,t˗/`>A1=@if:4hPSsNV T>ݥK/ecC,ńao@}=zYTy}U~Kl|Msˌ,[ !ҢE ӭ[7):,;v0.{=| x\Re$n)TTT|qSλ-[ 6dM6 ?;wJL/~F#6mT&M͛mfdiF,?]pfժU¸Nѕ;c]v(.fNG/qF 8P\(1^9PrK4Q'~g7PQMm>|п:rBbvx 77T@h4C[/!筟hĴlReB 9I P:uz<H,H[MjƍR~>}ٚDq?^z+J~ Uʌ@yi!`3a„+V8V9ַͱJ3b *XBPǦtlә8q\ ; 3tP*Ofdsrwuӿdž9LԱKwquʖ{bt>/;hԱa]ʵdN߾}|26|$.{:gmU<@c| A}'ls:Or۵k'%{夥ԝ^gABey]=o~םw"%9g#*_*xYʺA!Vى+½E 98_޸mbyI'y&id1 /\0bߎٗ?*mao*( gX\!.X_.̊H-I2HngϞfn1^q&C?:A!V_ʞJAg1h)qB`РAA8V l\\lr⦂ؼr9b ?e˖~-͛;6αJO]T,\wkpx?ֈ"p[:yq{qo&'e{[WXvU֒ ϵV[ܹs* .B3&vY^2s?Wpщo-9%(֭[g[>X5_PĤuvsu%OW%O(#I|2f5ʱ{q].F姞zjrӴF۽I.?)Fw8(jϢZwj׊9AlK1轵#y{)B؛ 7$Jb€BX 1@#ۂ[Jqy=#6(c̘1r]>ԇD- k"y~<[rԫIFVZڎ;"UkpĵNh w ~ KkTC 2𣞔QQщz*%2䞵QF9_?6f|\r 3ŋ,o֬YH89#8 K9R-#_PSW",( rBJKK%yϻQU-5QޡC1;={aSU|E@pja‚4ɋ /er@+7 1LIĒ:^B@.`HN3^ؒmNqaQWT V L" Q:$Bpi۶lVf$an t 1eʱi&Wء ca5w}eN- gb= =GJf(OԆʁrM ߴ⛠*5{[F3.]4Z&avYeGK/@wFH\(=r"(^xfٱMOL 8AT V9DŽ СCͤIӛ.k7D@LoŝRQE U~ 536-*NLǴ# "`I2uGڮPGX1E8\9TP~Mnlq%K)<^vҩS847mԙ_GM\<?υzx8/OƶjiËAjyǍWce6hvZR*ӰGB&f,@DҥKj EO}oGr˅զep*UW&?@ZUѼጌ*ppZ"^M81"- !RU~©E b~nʊZ{nj֯_/QZEeiӢ0 I~N`P'`#!C$SɗիVjD@W0KG9t"3t"@a=>t旾1F%K5E/E]-[3fTg<ڷ`Gju}ޑ(%Qé)JNnG#/%3|p3uT{y+۶m3 }T_C`ȏǗo^x C%pPZKm~3ݱcGYZh"ӱcGӬYԟJUqԵ oʔ)ծq_Þ={Q9X6mjXG0ɥGaVT+`干= / 3: L%2P蛴#~N:Ɍ;6T(#S[۶mC7합K7@_?~A~ ѭl oRz%/!۷o7j*괎6&(0={_Kff=TES{VE O|nVӧOs7 ?#e9!U~ H ֭0!U(QF63k5lʯkFC5s1첋T›7ojx۷rao>pӟ$J^zuc?sG'hwO_G[͜9SXiXnҤIz&(Uz^{^ |y뭷͘1c̷-sG?0s|r䀚87# ardwc7nl7onpi>CK Cfo л Ӭ^ZUVkך> GPl6l l%cU+IlْQ泌~\*bVΝ;"OW*!3?ԒbWXW^ feE꺤0;ZB0s0QD74\'Z$:ڵ( ړ2+U`!!Y XWX!3fV(Keq#*ŒY($7њYkoN8Y>G1h* 6#(dQQFwU~ mYy淿c=̏cQzALϘ1CyL_zs1{ ( EJ6j#6wС'##GK.4h ڃN4{[88k-A!]f58 CC???3'O7fԨQ}fʔ)aԘĥ\Hi; @xwa:u$$~aK/W#=ڼ 830KǼϯ\y商,yO; @̻X[haƏo^~eK.榛nJ-*jZps/3?XMR88n38vXÌ4&QWwF,#8(_uUbrh"aC4($."\$41+s=vm78q|K+wޢ }*:Ũi/"̚5 4He|IY=4*4rx뭷_BӾŸ'n:D"GtȻ:%6D-"`[ի,wRP x !wc$N NZC)Ucǎc)E3%"rJ17N"BJ,&)iG(Qؗ.\(ssL5gjfL@=^{I({iAKtAfn0g}'~|BD Kގu`LEԱ!b:-ZN>dRK9Y~uU'AջuV*qWU͎B0Ī(Bw^c#.̫-[ԯ_߱.9׿xۃ:6_I(Sj-|[+ tw}k-kVIxbpC7O>%z 4# 81ҥK=n@r f*z)qFӧOZ!ؓ&M[N*}'$Dp@]xBm@6'LU|3^`=R(-ԩSEAAğK+ƤŬ-^ۙ<'Q`CDJZ"O?T,q^N}ݎaԎ۷owݱuK/mYdLlT\veGYd<*YۜBXg_ %_Z|;E[f`{'dFɍ.Oeܹ\qzɧx˜[?<{m$H9ڒ_C*F-:/x]weM&Gt.H. 29yK{1(P_ IXZ`]f {a&"\|*F$/"ZH MZg{ U~hX"^Qnך{ǴnZ(IDC>`;,qrNV䜦NI_a&810я~$ ,(AAyeV ;>{I8@}<ZJ ڴiԩׯ$2#tMDzUT[{ .&R#=B Q֯*¯+@P>r!*@y⯵+@PW&ZE@(/Q #*6IENDB`pymongo-3.6.1/doc/static/sidebar.js0000644000076600000240000001421413245617773017457 0ustar shanestaff00000000000000/* * sidebar.js * ~~~~~~~~~~ * * This script makes the Sphinx sidebar collapsible and implements intelligent * scrolling. * * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to * collapse and expand the sidebar. * * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the * width of the sidebar and the margin-left of the document are decreased. * When the sidebar is expanded the opposite happens. This script saves a * per-browser/per-session cookie used to remember the position of the sidebar * among the pages. Once the browser is closed the cookie is deleted and the * position reset to the default (expanded). * * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ $(function() { // global elements used by the functions. // the 'sidebarbutton' element is defined as global after its // creation, in the add_sidebar_button function var jwindow = $(window); var jdocument = $(document); var bodywrapper = $('.bodywrapper'); var sidebar = $('.sphinxsidebar'); var sidebarwrapper = $('.sphinxsidebarwrapper'); // original margin-left of the bodywrapper and width of the sidebar // with the sidebar expanded var bw_margin_expanded = bodywrapper.css('margin-left'); var ssb_width_expanded = sidebar.width(); // margin-left of the bodywrapper and width of the sidebar // with the sidebar collapsed var bw_margin_collapsed = '.8em'; var ssb_width_collapsed = '.8em'; // colors used by the current theme var dark_color = '#AAAAAA'; var light_color = '#CCCCCC'; function get_viewport_height() { if (window.innerHeight) return window.innerHeight; else return jwindow.height(); } function sidebar_is_collapsed() { return sidebarwrapper.is(':not(:visible)'); } function toggle_sidebar() { if (sidebar_is_collapsed()) expand_sidebar(); else collapse_sidebar(); // adjust the scrolling of the sidebar scroll_sidebar(); } function collapse_sidebar() { sidebarwrapper.hide(); sidebar.css('width', ssb_width_collapsed); bodywrapper.css('margin-left', bw_margin_collapsed); sidebarbutton.css({ 'margin-left': '0', 'height': bodywrapper.height(), 'border-radius': '5px' }); sidebarbutton.find('span').text('»'); sidebarbutton.attr('title', _('Expand sidebar')); document.cookie = 'sidebar=collapsed'; } function expand_sidebar() { bodywrapper.css('margin-left', bw_margin_expanded); sidebar.css('width', ssb_width_expanded); sidebarwrapper.show(); sidebarbutton.css({ 'margin-left': ssb_width_expanded-12, 'height': bodywrapper.height(), 'border-radius': '0 5px 5px 0' }); sidebarbutton.find('span').text('«'); sidebarbutton.attr('title', _('Collapse sidebar')); //sidebarwrapper.css({'padding-top': // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); document.cookie = 'sidebar=expanded'; } function add_sidebar_button() { sidebarwrapper.css({ 'float': 'left', 'margin-right': '0', 'width': ssb_width_expanded - 28 }); // create the button sidebar.append( '
«
' ); var sidebarbutton = $('#sidebarbutton'); // find the height of the viewport to center the '<<' in the page var viewport_height = get_viewport_height(); var sidebar_offset = sidebar.offset().top; var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); sidebarbutton.find('span').css({ 'display': 'block', 'position': 'fixed', 'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 }); sidebarbutton.click(toggle_sidebar); sidebarbutton.attr('title', _('Collapse sidebar')); sidebarbutton.css({ 'border-radius': '0 5px 5px 0', 'color': '#444444', 'background-color': '#CCCCCC', 'font-size': '1.2em', 'cursor': 'pointer', 'height': sidebar_height, 'padding-top': '1px', 'padding-left': '1px', 'margin-left': ssb_width_expanded - 12 }); sidebarbutton.hover( function () { $(this).css('background-color', dark_color); }, function () { $(this).css('background-color', light_color); } ); } function set_position_from_cookie() { if (!document.cookie) return; var items = document.cookie.split(';'); for(var k=0; k wintop && curbot > winbot) { sidebarwrapper.css('top', $u.max([wintop - offset - 10, 0])); } else if (curtop < wintop && curbot < winbot) { sidebarwrapper.css('top', $u.min([winbot - sidebar_height - offset - 20, jdocument.height() - sidebar_height - 200])); } } } jwindow.scroll(scroll_sidebar); }); pymongo-3.6.1/doc/examples/0000755000076600000240000000000013246104133016013 5ustar shanestaff00000000000000pymongo-3.6.1/doc/examples/authentication.rst0000644000076600000240000002001213245621354021567 0ustar shanestaff00000000000000Authentication Examples ======================= MongoDB supports several different authentication mechanisms. These examples cover all authentication methods currently supported by PyMongo, documenting Python module and MongoDB version dependencies. Percent-Escaping Username and Password -------------------------------------- Username and password must be percent-escaped with :meth:`urllib.parse.quote_plus` in Python 3, or :meth:`urllib.quote_plus` in Python 2, to be used in a MongoDB URI. For example, in Python 3:: >>> from pymongo import MongoClient >>> import urllib.parse >>> username = urllib.parse.quote_plus('user') >>> username 'user' >>> password = urllib.parse.quote_plus('pass/word') >>> password 'pass%2Fword' >>> MongoClient('mongodb://%s:%s@127.0.0.1' % (username, password)) ... SCRAM-SHA-1 (RFC 5802) ---------------------- .. versionadded:: 2.8 SCRAM-SHA-1 is the default authentication mechanism supported by a cluster configured for authentication with MongoDB 3.0 or later. Authentication requires a username, a password, and a database name. The default database name is "admin", this can be overidden with the ``authSource`` option. Credentials can be specified as arguments to :class:`~pymongo.mongo_client.MongoClient`:: >>> from pymongo import MongoClient >>> client = MongoClient('example.com', ... username='user', ... password='password', ... authSource='the_database', ... authMechanism='SCRAM-SHA-1') Or through the MongoDB URI:: >>> uri = "mongodb://user:password@example.com/the_database?authMechanism=SCRAM-SHA-1" >>> client = MongoClient(uri) For best performance on Python versions older than 2.7.8 install `backports.pbkdf2`_. .. _backports.pbkdf2: https://pypi.python.org/pypi/backports.pbkdf2/ MONGODB-CR ---------- Before MongoDB 3.0 the default authentication mechanism was MONGODB-CR, the "MongoDB Challenge-Response" protocol:: >>> from pymongo import MongoClient >>> client = MongoClient('example.com', ... username='user', ... password='password', ... authMechanism='MONGODB-CR') >>> >>> uri = "mongodb://user:password@example.com/the_database?authMechanism=MONGODB-CR" >>> client = MongoClient(uri) Default Authentication Mechanism -------------------------------- If no mechanism is specified, PyMongo automatically uses MONGODB-CR when connected to a pre-3.0 version of MongoDB, and SCRAM-SHA-1 when connected to a recent version. Default Database and "authSource" --------------------------------- You can specify both a default database and the authentication database in the URI:: >>> uri = "mongodb://user:password@example.com/default_db?authSource=admin" >>> client = MongoClient(uri) PyMongo will authenticate on the "admin" database, but the default database will be "default_db":: >>> # get_database with no "name" argument chooses the DB from the URI >>> db = MongoClient(uri).get_database() >>> print(db.name) 'default_db' MONGODB-X509 ------------ .. versionadded:: 2.6 The MONGODB-X509 mechanism authenticates a username derived from the distinguished subject name of the X.509 certificate presented by the driver during SSL negotiation. This authentication method requires the use of SSL connections with certificate validation and is available in MongoDB 2.6 and newer:: >>> import ssl >>> from pymongo import MongoClient >>> client = MongoClient('example.com', ... username="" ... authMechanism="MONGODB-X509", ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_cert_reqs=ssl.CERT_REQUIRED, ... ssl_ca_certs='/path/to/ca.pem') MONGODB-X509 authenticates against the $external virtual database, so you do not have to specify a database in the URI:: >>> uri = "mongodb://@example.com/?authMechanism=MONGODB-X509" >>> client = MongoClient(uri, ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_cert_reqs=ssl.CERT_REQUIRED, ... ssl_ca_certs='/path/to/ca.pem') >>> .. versionchanged:: 3.4 When connected to MongoDB >= 3.4 the username is no longer required. .. _use_kerberos: GSSAPI (Kerberos) ----------------- .. versionadded:: 2.5 GSSAPI (Kerberos) authentication is available in the Enterprise Edition of MongoDB. Unix ~~~~ To authenticate using GSSAPI you must first install the python `kerberos`_ or `pykerberos`_ module using easy_install or pip. Make sure you run kinit before using the following authentication methods:: $ kinit mongodbuser@EXAMPLE.COM mongodbuser@EXAMPLE.COM's Password: $ klist Credentials cache: FILE:/tmp/krb5cc_1000 Principal: mongodbuser@EXAMPLE.COM Issued Expires Principal Feb 9 13:48:51 2013 Feb 9 23:48:51 2013 krbtgt/EXAMPLE.COM@EXAMPLE.COM Now authenticate using the MongoDB URI. GSSAPI authenticates against the $external virtual database so you do not have to specify a database in the URI:: >>> # Note: the kerberos principal must be url encoded. >>> from pymongo import MongoClient >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@mongo-server.example.com/?authMechanism=GSSAPI" >>> client = MongoClient(uri) >>> The default service name used by MongoDB and PyMongo is `mongodb`. You can specify a custom service name with the ``authMechanismProperties`` option:: >>> from pymongo import MongoClient >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@mongo-server.example.com/?authMechanism=GSSAPI&authMechanismProperties=SERVICE_NAME:myservicename" >>> client = MongoClient(uri) Windows (SSPI) ~~~~~~~~~~~~~~ .. versionadded:: 3.3 First install the `winkerberos`_ module. Unlike authentication on Unix kinit is not used. If the user to authenticate is different from the user that owns the application process provide a password to authenticate:: >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM:mongodbuserpassword@example.com/?authMechanism=GSSAPI" Two extra ``authMechanismProperties`` are supported on Windows platforms: - CANONICALIZE_HOST_NAME - Uses the fully qualified domain name (FQDN) of the MongoDB host for the server principal (GSSAPI libraries on Unix do this by default):: >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@example.com/?authMechanism=GSSAPI&authMechanismProperties=CANONICALIZE_HOST_NAME:true" - SERVICE_REALM - This is used when the user's realm is different from the service's realm:: >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@example.com/?authMechanism=GSSAPI&authMechanismProperties=SERVICE_REALM:otherrealm" .. _kerberos: http://pypi.python.org/pypi/kerberos .. _pykerberos: https://pypi.python.org/pypi/pykerberos .. _winkerberos: https://pypi.python.org/pypi/winkerberos/ SASL PLAIN (RFC 4616) --------------------- .. versionadded:: 2.6 MongoDB Enterprise Edition version 2.6 and newer support the SASL PLAIN authentication mechanism, initially intended for delegating authentication to an LDAP server. Using the PLAIN mechanism is very similar to MONGODB-CR. These examples use the $external virtual database for LDAP support:: >>> from pymongo import MongoClient >>> uri = "mongodb://user:password@example.com/?authMechanism=PLAIN&authSource=$external" >>> client = MongoClient(uri) >>> SASL PLAIN is a clear-text authentication mechanism. We **strongly** recommend that you connect to MongoDB using SSL with certificate validation when using the SASL PLAIN mechanism:: >>> import ssl >>> from pymongo import MongoClient >>> uri = "mongodb://user:password@example.com/?authMechanism=PLAIN&authSource=$external" >>> client = MongoClient(uri, ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_cert_reqs=ssl.CERT_REQUIRED, ... ssl_ca_certs='/path/to/ca.pem') >>> pymongo-3.6.1/doc/examples/index.rst0000644000076600000240000000115113245617773017674 0ustar shanestaff00000000000000Examples ======== The examples in this section are intended to give in depth overviews of how to accomplish specific tasks with MongoDB and PyMongo. Unless otherwise noted, all examples assume that a MongoDB instance is running on the default host and port. Assuming you have `downloaded and installed `_ MongoDB, you can start it like so: .. code-block:: bash $ mongod .. toctree:: :maxdepth: 1 aggregation authentication collations copydb bulk datetimes geo gevent gridfs high_availability mod_wsgi tailable tls pymongo-3.6.1/doc/examples/datetimes.rst0000644000076600000240000000745113245617773020555 0ustar shanestaff00000000000000Datetimes and Timezones ======================= .. testsetup:: import datetime from pymongo import MongoClient from bson.codec_options import CodecOptions client = MongoClient() client.drop_database('dt_example') db = client.dt_example These examples show how to handle Python :class:`datetime.datetime` objects correctly in PyMongo. Basic Usage ----------- PyMongo uses :class:`datetime.datetime` objects for representing dates and times in MongoDB documents. Because MongoDB assumes that dates and times are in UTC, care should be taken to ensure that dates and times written to the database reflect UTC. For example, the following code stores the current UTC date and time into MongoDB: .. doctest:: >>> result = db.objects.insert_one( ... {"last_modified": datetime.datetime.utcnow()}) Always use :meth:`datetime.datetime.utcnow`, which returns the current time in UTC, instead of :meth:`datetime.datetime.now`, which returns the current local time. Avoid doing this: .. doctest:: >>> result = db.objects.insert_one( ... {"last_modified": datetime.datetime.now()}) The value for `last_modified` is very different between these two examples, even though both documents were stored at around the same local time. This will be confusing to the application that reads them: .. doctest:: >>> [doc['last_modified'] for doc in db.objects.find()] # doctest: +SKIP [datetime.datetime(2015, 7, 8, 18, 17, 28, 324000), datetime.datetime(2015, 7, 8, 11, 17, 42, 911000)] :class:`bson.codec_options.CodecOptions` has a `tz_aware` option that enables "aware" :class:`datetime.datetime` objects, i.e., datetimes that know what timezone they're in. By default, PyMongo retrieves naive datetimes: .. doctest:: >>> result = db.tzdemo.insert_one( ... {'date': datetime.datetime(2002, 10, 27, 6, 0, 0)}) >>> db.tzdemo.find_one()['date'] datetime.datetime(2002, 10, 27, 6, 0) >>> options = CodecOptions(tz_aware=True) >>> db.get_collection('tzdemo', codec_options=options).find_one()['date'] # doctest: +SKIP datetime.datetime(2002, 10, 27, 6, 0, tzinfo=) Saving Datetimes with Timezones ------------------------------- When storing :class:`datetime.datetime` objects that specify a timezone (i.e. they have a `tzinfo` property that isn't ``None``), PyMongo will convert those datetimes to UTC automatically: .. doctest:: >>> import pytz >>> pacific = pytz.timezone('US/Pacific') >>> aware_datetime = pacific.localize( ... datetime.datetime(2002, 10, 27, 6, 0, 0)) >>> result = db.times.insert_one({"date": aware_datetime}) >>> db.times.find_one()['date'] datetime.datetime(2002, 10, 27, 14, 0) Reading Time ------------ As previously mentioned, by default all :class:`datetime.datetime` objects returned by PyMongo will be naive but reflect UTC (i.e. the time as stored in MongoDB). By setting the `tz_aware` option on :class:`~bson.codec_options.CodecOptions`, :class:`datetime.datetime` objects will be timezone-aware and have a `tzinfo` property that reflects the UTC timezone. PyMongo 3.1 introduced a `tzinfo` property that can be set on :class:`~bson.codec_options.CodecOptions` to convert :class:`datetime.datetime` objects to local time automatically. For example, if we wanted to read all times out of MongoDB in US/Pacific time: >>> from bson.codec_options import CodecOptions >>> db.times.find_one()['date'] datetime.datetime(2002, 10, 27, 14, 0) >>> aware_times = db.times.with_options(codec_options=CodecOptions( ... tz_aware=True, ... tzinfo=pytz.timezone('US/Pacific'))) >>> result = aware_times.find_one() datetime.datetime(2002, 10, 27, 6, 0, # doctest: +NORMALIZE_WHITESPACE tzinfo=) pymongo-3.6.1/doc/examples/copydb.rst0000644000076600000240000000277013245617773020055 0ustar shanestaff00000000000000Copying a Database ================== To copy a database within a single mongod process, or between mongod servers, simply connect to the target mongod and use the :meth:`~pymongo.database.Database.command` method:: >>> from pymongo import MongoClient >>> client = MongoClient('target.example.com') >>> client.admin.command('copydb', fromdb='source_db_name', todb='target_db_name') To copy from a different mongod server that is not password-protected:: >>> client.admin.command('copydb', fromdb='source_db_name', todb='target_db_name', fromhost='source.example.com') If the target server is password-protected, authenticate to the "admin" database:: >>> client = MongoClient('target.example.com', ... username='administrator', ... password='pwd') >>> client.admin.command('copydb', fromdb='source_db_name', todb='target_db_name', fromhost='source.example.com') See the :doc:`authentication examples `. If the **source** server is password-protected, use the `copyDatabase function in the mongo shell`_. Versions of PyMongo before 3.0 included a ``copy_database`` helper method, but it has been removed. .. _copyDatabase function in the mongo shell: http://docs.mongodb.org/manual/reference/method/db.copyDatabase/ pymongo-3.6.1/doc/examples/tls.rst0000644000076600000240000001364013245621354017363 0ustar shanestaff00000000000000TLS/SSL and PyMongo =================== PyMongo supports connecting to MongoDB over TLS/SSL. This guide covers the configuration options supported by PyMongo. See `the server documentation `_ to configure MongoDB. Dependencies ............ For connections using TLS/SSL, PyMongo may require third party dependencies as determined by your version of Python. With PyMongo 3.3+, you can install PyMongo 3.3+ and any TLS/SSL-related dependencies using the following pip command:: $ python -m pip install pymongo[tls] Earlier versions of PyMongo require you to manually install the dependencies listed below. Python 2.x `````````` The `ipaddress`_ module is required on all platforms. When using CPython < 2.7.9 or PyPy < 2.5.1: - On Windows, the `wincertstore`_ module is required. - On all other platforms, the `certifi`_ module is required. Python 3.x `````````` On Windows, the `wincertstore`_ module is required when using PyPy3 < 3.5. .. _ipaddress: https://pypi.python.org/pypi/ipaddress .. _wincertstore: https://pypi.python.org/pypi/wincertstore .. _certifi: https://pypi.python.org/pypi/certifi .. warning:: Industry best practices, and some regulations, require the use of TLS 1.1 or newer. Though no application changes are required for PyMongo to make use of the newest protocols, some operating systems or versions may not provide an OpenSSL version new enough to support them. Users of macOS older than 10.13 (High Sierra) will need to install Python from `python.org`_, `homebrew`_, `macports`_, or another similar source. Users of Linux or other non-macOS Unix can check their OpenSSL version like this:: $ openssl version If the version number is less than 1.0.1 support for TLS 1.1 or newer is not available. Contact your operating system vendor for a solution or upgrade to a newer distribution. You can check your Python interpreter by installing the `requests`_ module and executing the following command:: python -c "import requests; print(requests.get('https://www.howsmyssl.com/a/check', verify=False).json()['tls_version'])" You should see "TLS 1.X" where X is >= 1. You can read more about TLS versions and their security implications here: ``_ .. _python.org: https://www.python.org/downloads/ .. _homebrew: https://brew.sh/ .. _macports: https://www.macports.org/ .. _requests: https://pypi.python.org/pypi/requests Basic configuration ................... In many cases connecting to MongoDB over TLS/SSL requires nothing more than passing ``ssl=True`` as a keyword argument to :class:`~pymongo.mongo_client.MongoClient`:: >>> client = pymongo.MongoClient('example.com', ssl=True) Or passing ``ssl=true`` in the URI:: >>> client = pymongo.MongoClient('mongodb://example.com/?ssl=true') This configures PyMongo to connect to the server using TLS, verify the server's certificate and verify that the host you are attempting to connect to is listed by that certificate. Certificate verification policy ............................... By default, PyMongo is configured to require a certificate from the server when TLS is enabled. This is configurable using the `ssl_cert_reqs` option. To disable this requirement pass ``ssl.CERT_NONE`` as a keyword parameter:: >>> import ssl >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_cert_reqs=ssl.CERT_NONE) Or, in the URI:: >>> uri = 'mongodb://example.com/?ssl=true&ssl_cert_reqs=CERT_NONE' >>> client = pymongo.MongoClient(uri) Specifying a CA file .................... In some cases you may want to configure PyMongo to use a specific set of CA certificates. This is most often the case when using "self-signed" server certificates. The `ssl_ca_certs` option takes a path to a CA file. It can be passed as a keyword argument:: >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_ca_certs='/path/to/ca.pem') Or, in the URI:: >>> uri = 'mongodb://example.com/?ssl=true&ssl_ca_certs=/path/to/ca.pem' >>> client = pymongo.MongoClient(uri) Specifying a certificate revocation list ........................................ Python 2.7.9+ (pypy 2.5.1+) and 3.4+ provide support for certificate revocation lists. The `ssl_crlfile` option takes a path to a CRL file. It can be passed as a keyword argument:: >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_crlfile='/path/to/crl.pem') Or, in the URI:: >>> uri = 'mongodb://example.com/?ssl=true&ssl_crlfile=/path/to/crl.pem' >>> client = pymongo.MongoClient(uri) Client certificates ................... PyMongo can be configured to present a client certificate using the `ssl_certfile` option:: >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_certfile='/path/to/client.pem') If the private key for the client certificate is stored in a separate file use the `ssl_keyfile` option:: >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_keyfile='/path/to/key.pem') Python 2.7.9+ (pypy 2.5.1+) and 3.3+ support providing a password or passphrase to decrypt encrypted private keys. Use the `ssl_pem_passphrase` option:: >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_keyfile='/path/to/key.pem', ... ssl_pem_passphrase=) These options can also be passed as part of the MongoDB URI. pymongo-3.6.1/doc/examples/tailable.rst0000644000076600000240000000307013245621354020332 0ustar shanestaff00000000000000Tailable Cursors ================ By default, MongoDB will automatically close a cursor when the client has exhausted all results in the cursor. However, for `capped collections `_ you may use a `tailable cursor `_ that remains open after the client exhausts the results in the initial cursor. The following is a basic example of using a tailable cursor to tail the oplog of a replica set member:: import time import pymongo client = pymongo.MongoClient() oplog = client.local.oplog.rs first = oplog.find().sort('$natural', pymongo.ASCENDING).limit(-1).next() print(first) ts = first['ts'] while True: # For a regular capped collection CursorType.TAILABLE_AWAIT is the # only option required to create a tailable cursor. When querying the # oplog the oplog_replay option enables an optimization to quickly # find the 'ts' value we're looking for. The oplog_replay option # can only be used when querying the oplog. cursor = oplog.find({'ts': {'$gt': ts}}, cursor_type=pymongo.CursorType.TAILABLE_AWAIT, oplog_replay=True) while cursor.alive: for doc in cursor: ts = doc['ts'] print(doc) # We end up here if the find() returned no documents or if the # tailable cursor timed out (no new documents were added to the # collection for more than 1 second). time.sleep(1) pymongo-3.6.1/doc/examples/collations.rst0000644000076600000240000001147413245617773020745 0ustar shanestaff00000000000000Collations ========== .. seealso:: The API docs for :mod:`~pymongo.collation`. Collations are a new feature in MongoDB version 3.4. They provide a set of rules to use when comparing strings that comply with the conventions of a particular language, such as Spanish or German. If no collation is specified, the server sorts strings based on a binary comparison. Many languages have specific ordering rules, and collations allow users to build applications that adhere to language-specific comparison rules. In French, for example, the last accent in a given word determines the sorting order. The correct sorting order for the following four words in French is:: cote < côte < coté < côté Specifying a French collation allows users to sort string fields using the French sort order. Usage ----- Users can specify a collation for a :ref:`collection`, an :ref:`index`, or a :ref:`CRUD command `. Collation Parameters: ~~~~~~~~~~~~~~~~~~~~~ Collations can be specified with the :class:`~pymongo.collation.Collation` model or with plain Python dictionaries. The structure is the same:: Collation(locale=, caseLevel=, caseFirst=, strength=, numericOrdering=, alternate=, maxVariable=, backwards=) The only required parameter is ``locale``, which the server parses as an `ICU format locale ID `_. For example, set ``locale`` to ``en_US`` to represent US English or ``fr_CA`` to represent Canadian French. For a complete description of the available parameters, see the MongoDB `manual `_. .. COMMENT add link for manual entry. .. _collation-on-collection: Assign a Default Collation to a Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The following example demonstrates how to create a new collection called ``contacts`` and assign a default collation with the ``fr_CA`` locale. This operation ensures that all queries that are run against the ``contacts`` collection use the ``fr_CA`` collation unless another collation is explicitly specified:: from pymongo import MongoClient from pymongo.collation import Collation db = MongoClient().test collection = db.create_collection('contacts', collation=Collation(locale='fr_CA')) .. _collation-on-index: Assign a Default Collation to an Index ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When creating a new index, you can specify a default collation. The following example shows how to create an index on the ``name`` field of the ``contacts`` collection, with the ``unique`` parameter enabled and a default collation with ``locale`` set to ``fr_CA``:: from pymongo import MongoClient from pymongo.collation import Collation contacts = MongoClient().test.contacts contacts.create_index('name', unique=True, collation=Collation(locale='fr_CA')) .. _collation-on-operation: Specify a Collation for a Query ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Individual queries can specify a collation to use when sorting results. The following example demonstrates a query that runs on the ``contacts`` collection in database ``test``. It matches on documents that contain ``New York`` in the ``city`` field, and sorts on the ``name`` field with the ``fr_CA`` collation:: from pymongo import MongoClient from pymongo.collation import Collation collection = MongoClient().test.contacts docs = collection.find({'city': 'New York'}).sort('name').collation( Collation(locale='fr_CA')) Other Query Types ~~~~~~~~~~~~~~~~~ You can use collations to control document matching rules for several different types of queries. All the various update and delete methods (:meth:`~pymongo.collection.Collection.update_one`, :meth:`~pymongo.collection.Collection.update_many`, :meth:`~pymongo.collection.Collection.delete_one`, etc.) support collation, and you can create query filters which employ collations to comply with any of the languages and variants available to the ``locale`` parameter. The following example uses a collation with ``strength`` set to :const:`~pymongo.collation.CollationStrength.SECONDARY`, which considers only the base character and character accents in string comparisons, but not case sensitivity, for example. All documents in the ``contacts`` collection with ``jürgen`` (case-insensitive) in the ``first_name`` field are updated:: from pymongo import MongoClient from pymongo.collation import Collation, CollationStrength contacts = MongoClient().test.contacts result = contacts.update_many( {'first_name': 'jürgen'}, {'$set': {'verified': 1}}, collation=Collation(locale='de', strength=CollationStrength.SECONDARY)) pymongo-3.6.1/doc/examples/mod_wsgi.rst0000644000076600000240000000507713245617773020410 0ustar shanestaff00000000000000.. _pymongo-and-mod_wsgi: PyMongo and mod_wsgi ==================== To run your application under `mod_wsgi `_, follow these guidelines: * Run ``mod_wsgi`` in daemon mode with the ``WSGIDaemonProcess`` directive. * Assign each application to a separate daemon with ``WSGIProcessGroup``. * Use ``WSGIApplicationGroup %{GLOBAL}`` to ensure your application is running in the daemon's main Python interpreter, not a sub interpreter. For example, this ``mod_wsgi`` configuration ensures an application runs in the main interpreter:: WSGIDaemonProcess my_process WSGIScriptAlias /my_app /path/to/app.wsgi WSGIProcessGroup my_process WSGIApplicationGroup %{GLOBAL} If you have multiple applications that use PyMongo, put each in a separate daemon, still in the global application group:: WSGIDaemonProcess my_process WSGIScriptAlias /my_app /path/to/app.wsgi WSGIProcessGroup my_process WSGIDaemonProcess my_other_process WSGIScriptAlias /my_other_app /path/to/other_app.wsgi WSGIProcessGroup my_other_process WSGIApplicationGroup %{GLOBAL} Background: ``mod_wsgi`` can run in "embedded" mode when only WSGIScriptAlias is set, or "daemon" mode with WSGIDaemonProcess. In daemon mode, ``mod_wsgi`` can run your application in the Python main interpreter, or in sub interpreters. The correct way to run a PyMongo application is in daemon mode, using the main interpreter. Python C extensions in general have issues running in multiple Python sub interpreters. These difficulties are explained in the documentation for `Py_NewInterpreter `_ and in the `Multiple Python Sub Interpreters `_ section of the ``mod_wsgi`` documentation. Beginning with PyMongo 2.7, the C extension for BSON detects when it is running in a sub interpreter and activates a workaround, which adds a small cost to BSON decoding. To avoid this cost, use ``WSGIApplicationGroup %{GLOBAL}`` to ensure your application runs in the main interpreter. Since your program runs in the main interpreter it should not share its process with any other applications, lest they interfere with each other's state. Each application should have its own daemon process, as shown in the example above. pymongo-3.6.1/doc/examples/gridfs.rst0000644000076600000240000000437313234134477020045 0ustar shanestaff00000000000000GridFS Example ============== .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('gridfs_example') This example shows how to use :mod:`gridfs` to store large binary objects (e.g. files) in MongoDB. .. seealso:: The API docs for :mod:`gridfs`. .. seealso:: `This blog post `_ for some motivation behind this API. Setup ----- We start by creating a :class:`~gridfs.GridFS` instance to use: .. doctest:: >>> from pymongo import MongoClient >>> import gridfs >>> >>> db = MongoClient().gridfs_example >>> fs = gridfs.GridFS(db) Every :class:`~gridfs.GridFS` instance is created with and will operate on a specific :class:`~pymongo.database.Database` instance. Saving and Retrieving Data -------------------------- The simplest way to work with :mod:`gridfs` is to use its key/value interface (the :meth:`~gridfs.GridFS.put` and :meth:`~gridfs.GridFS.get` methods). To write data to GridFS, use :meth:`~gridfs.GridFS.put`: .. doctest:: >>> a = fs.put(b"hello world") :meth:`~gridfs.GridFS.put` creates a new file in GridFS, and returns the value of the file document's ``"_id"`` key. Given that ``"_id"`` we can use :meth:`~gridfs.GridFS.get` to get back the contents of the file: .. doctest:: >>> fs.get(a).read() 'hello world' :meth:`~gridfs.GridFS.get` returns a file-like object, so we get the file's contents by calling :meth:`~gridfs.grid_file.GridOut.read`. In addition to putting a :class:`str` as a GridFS file, we can also put any file-like object (an object with a :meth:`read` method). GridFS will handle reading the file in chunk-sized segments automatically. We can also add additional attributes to the file as keyword arguments: .. doctest:: >>> b = fs.put(fs.get(a), filename="foo", bar="baz") >>> out = fs.get(b) >>> out.read() 'hello world' >>> out.filename u'foo' >>> out.bar u'baz' >>> out.upload_date datetime.datetime(...) The attributes we set in :meth:`~gridfs.GridFS.put` are stored in the file document, and retrievable after calling :meth:`~gridfs.GridFS.get`. Some attributes (like ``"filename"``) are special and are defined in the GridFS specification - see that document for more details. pymongo-3.6.1/doc/examples/aggregation.rst0000644000076600000240000001345713245621354021056 0ustar shanestaff00000000000000Aggregation Examples ==================== There are several methods of performing aggregations in MongoDB. These examples cover the new aggregation framework, using map reduce and using the group method. .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('aggregation_example') Setup ----- To start, we'll insert some example data which we can perform aggregations on: .. doctest:: >>> from pymongo import MongoClient >>> db = MongoClient().aggregation_example >>> result = db.things.insert_many([{"x": 1, "tags": ["dog", "cat"]}, ... {"x": 2, "tags": ["cat"]}, ... {"x": 2, "tags": ["mouse", "cat", "dog"]}, ... {"x": 3, "tags": []}]) >>> result.inserted_ids [ObjectId('...'), ObjectId('...'), ObjectId('...'), ObjectId('...')] .. _aggregate-examples: Aggregation Framework --------------------- This example shows how to use the :meth:`~pymongo.collection.Collection.aggregate` method to use the aggregation framework. We'll perform a simple aggregation to count the number of occurrences for each tag in the ``tags`` array, across the entire collection. To achieve this we need to pass in three operations to the pipeline. First, we need to unwind the ``tags`` array, then group by the tags and sum them up, finally we sort by count. As python dictionaries don't maintain order you should use :class:`~bson.son.SON` or :class:`collections.OrderedDict` where explicit ordering is required eg "$sort": .. note:: aggregate requires server version **>= 2.1.0**. .. doctest:: >>> from bson.son import SON >>> pipeline = [ ... {"$unwind": "$tags"}, ... {"$group": {"_id": "$tags", "count": {"$sum": 1}}}, ... {"$sort": SON([("count", -1), ("_id", -1)])} ... ] >>> import pprint >>> pprint.pprint(list(db.things.aggregate(pipeline))) [{u'_id': u'cat', u'count': 3}, {u'_id': u'dog', u'count': 2}, {u'_id': u'mouse', u'count': 1}] To run an explain plan for this aggregation use the :meth:`~pymongo.database.Database.command` method:: >>> db.command('aggregate', 'things', pipeline=pipeline, explain=True) {u'ok': 1.0, u'stages': [...]} As well as simple aggregations the aggregation framework provides projection capabilities to reshape the returned data. Using projections and aggregation, you can add computed fields, create new virtual sub-objects, and extract sub-fields into the top-level of results. .. seealso:: The full documentation for MongoDB's `aggregation framework `_ Map/Reduce ---------- Another option for aggregation is to use the map reduce framework. Here we will define **map** and **reduce** functions to also count the number of occurrences for each tag in the ``tags`` array, across the entire collection. Our **map** function just emits a single `(key, 1)` pair for each tag in the array: .. doctest:: >>> from bson.code import Code >>> mapper = Code(""" ... function () { ... this.tags.forEach(function(z) { ... emit(z, 1); ... }); ... } ... """) The **reduce** function sums over all of the emitted values for a given key: .. doctest:: >>> reducer = Code(""" ... function (key, values) { ... var total = 0; ... for (var i = 0; i < values.length; i++) { ... total += values[i]; ... } ... return total; ... } ... """) .. note:: We can't just return ``values.length`` as the **reduce** function might be called iteratively on the results of other reduce steps. Finally, we call :meth:`~pymongo.collection.Collection.map_reduce` and iterate over the result collection: .. doctest:: >>> result = db.things.map_reduce(mapper, reducer, "myresults") >>> for doc in result.find(): ... pprint.pprint(doc) ... {u'_id': u'cat', u'value': 3.0} {u'_id': u'dog', u'value': 2.0} {u'_id': u'mouse', u'value': 1.0} Advanced Map/Reduce ------------------- PyMongo's API supports all of the features of MongoDB's map/reduce engine. One interesting feature is the ability to get more detailed results when desired, by passing `full_response=True` to :meth:`~pymongo.collection.Collection.map_reduce`. This returns the full response to the map/reduce command, rather than just the result collection: .. doctest:: >>> pprint.pprint( ... db.things.map_reduce(mapper, reducer, "myresults", full_response=True)) {...u'counts': {u'emit': 6, u'input': 4, u'output': 3, u'reduce': 2}, u'ok': ..., u'result': u'...', u'timeMillis': ...} All of the optional map/reduce parameters are also supported, simply pass them as keyword arguments. In this example we use the `query` parameter to limit the documents that will be mapped over: .. doctest:: >>> results = db.things.map_reduce( ... mapper, reducer, "myresults", query={"x": {"$lt": 2}}) >>> for doc in results.find(): ... pprint.pprint(doc) ... {u'_id': u'cat', u'value': 1.0} {u'_id': u'dog', u'value': 1.0} You can use :class:`~bson.son.SON` or :class:`collections.OrderedDict` to specify a different database to store the result collection: .. doctest:: >>> from bson.son import SON >>> pprint.pprint( ... db.things.map_reduce( ... mapper, ... reducer, ... out=SON([("replace", "results"), ("db", "outdb")]), ... full_response=True)) {...u'counts': {u'emit': 6, u'input': 4, u'output': 3, u'reduce': 2}, u'ok': ..., u'result': {u'collection': ..., u'db': ...}, u'timeMillis': ...} .. seealso:: The full list of options for MongoDB's `map reduce engine `_ pymongo-3.6.1/doc/examples/high_availability.rst0000644000076600000240000003404013245617773022241 0ustar shanestaff00000000000000High Availability and PyMongo ============================= PyMongo makes it easy to write highly available applications whether you use a `single replica set `_ or a `large sharded cluster `_. Connecting to a Replica Set --------------------------- PyMongo makes working with `replica sets `_ easy. Here we'll launch a new replica set and show how to handle both initialization and normal connections with PyMongo. .. mongodoc:: rs Starting a Replica Set ~~~~~~~~~~~~~~~~~~~~~~ The main `replica set documentation `_ contains extensive information about setting up a new replica set or migrating an existing MongoDB setup, be sure to check that out. Here, we'll just do the bare minimum to get a three node replica set setup locally. .. warning:: Replica sets should always use multiple nodes in production - putting all set members on the same physical node is only recommended for testing and development. We start three ``mongod`` processes, each on a different port and with a different dbpath, but all using the same replica set name "foo". .. code-block:: bash $ mkdir -p /data/db0 /data/db1 /data/db2 $ mongod --port 27017 --dbpath /data/db0 --replSet foo .. code-block:: bash $ mongod --port 27018 --dbpath /data/db1 --replSet foo .. code-block:: bash $ mongod --port 27019 --dbpath /data/db2 --replSet foo Initializing the Set ~~~~~~~~~~~~~~~~~~~~ At this point all of our nodes are up and running, but the set has yet to be initialized. Until the set is initialized no node will become the primary, and things are essentially "offline". To initialize the set we need to connect to a single node and run the initiate command:: >>> from pymongo import MongoClient >>> c = MongoClient('localhost', 27017) .. note:: We could have connected to any of the other nodes instead, but only the node we initiate from is allowed to contain any initial data. After connecting, we run the initiate command to get things started:: >>> config = {'_id': 'foo', 'members': [ ... {'_id': 0, 'host': 'localhost:27017'}, ... {'_id': 1, 'host': 'localhost:27018'}, ... {'_id': 2, 'host': 'localhost:27019'}]} >>> c.admin.command("replSetInitiate", config) {'ok': 1.0, ...} The three ``mongod`` servers we started earlier will now coordinate and come online as a replica set. Connecting to a Replica Set ~~~~~~~~~~~~~~~~~~~~~~~~~~~ The initial connection as made above is a special case for an uninitialized replica set. Normally we'll want to connect differently. A connection to a replica set can be made using the :meth:`~pymongo.mongo_client.MongoClient` constructor, specifying one or more members of the set, along with the replica set name. Any of the following connects to the replica set we just created:: >>> MongoClient('localhost', replicaset='foo') MongoClient(host=['localhost:27017'], replicaset='foo', ...) >>> MongoClient('localhost:27018', replicaset='foo') MongoClient(['localhost:27018'], replicaset='foo', ...) >>> MongoClient('localhost', 27019, replicaset='foo') MongoClient(['localhost:27019'], replicaset='foo', ...) >>> MongoClient('mongodb://localhost:27017,localhost:27018/?replicaSet=foo') MongoClient(['localhost:27017', 'localhost:27018'], replicaset='foo', ...) The addresses passed to :meth:`~pymongo.mongo_client.MongoClient` are called the *seeds*. As long as at least one of the seeds is online, MongoClient discovers all the members in the replica set, and determines which is the current primary and which are secondaries or arbiters. Each seed must be the address of a single mongod. Multihomed and round robin DNS addresses are **not** supported. The :class:`~pymongo.mongo_client.MongoClient` constructor is non-blocking: the constructor returns immediately while the client connects to the replica set using background threads. Note how, if you create a client and immediately print the string representation of its :attr:`~pymongo.mongo_client.MongoClient.nodes` attribute, the list may be empty initially. If you wait a moment, MongoClient discovers the whole replica set:: >>> from time import sleep >>> c = MongoClient(replicaset='foo'); print(c.nodes); sleep(0.1); print(c.nodes) frozenset([]) frozenset([(u'localhost', 27019), (u'localhost', 27017), (u'localhost', 27018)]) You need not wait for replica set discovery in your application, however. If you need to do any operation with a MongoClient, such as a :meth:`~pymongo.collection.Collection.find` or an :meth:`~pymongo.collection.Collection.insert_one`, the client waits to discover a suitable member before it attempts the operation. Handling Failover ~~~~~~~~~~~~~~~~~ When a failover occurs, PyMongo will automatically attempt to find the new primary node and perform subsequent operations on that node. This can't happen completely transparently, however. Here we'll perform an example failover to illustrate how everything behaves. First, we'll connect to the replica set and perform a couple of basic operations:: >>> db = MongoClient("localhost", replicaSet='foo').test >>> db.test.insert_one({"x": 1}).inserted_id ObjectId('...') >>> db.test.find_one() {u'x': 1, u'_id': ObjectId('...')} By checking the host and port, we can see that we're connected to *localhost:27017*, which is the current primary:: >>> db.client.address ('localhost', 27017) Now let's bring down that node and see what happens when we run our query again:: >>> db.test.find_one() Traceback (most recent call last): pymongo.errors.AutoReconnect: ... We get an :class:`~pymongo.errors.AutoReconnect` exception. This means that the driver was not able to connect to the old primary (which makes sense, as we killed the server), but that it will attempt to automatically reconnect on subsequent operations. When this exception is raised our application code needs to decide whether to retry the operation or to simply continue, accepting the fact that the operation might have failed. On subsequent attempts to run the query we might continue to see this exception. Eventually, however, the replica set will failover and elect a new primary (this should take no more than a couple of seconds in general). At that point the driver will connect to the new primary and the operation will succeed:: >>> db.test.find_one() {u'x': 1, u'_id': ObjectId('...')} >>> db.client.address ('localhost', 27018) Bring the former primary back up. It will rejoin the set as a secondary. Now we can move to the next section: distributing reads to secondaries. .. _secondary-reads: Secondary Reads ~~~~~~~~~~~~~~~ By default an instance of MongoClient sends queries to the primary member of the replica set. To use secondaries for queries we have to change the read preference:: >>> client = MongoClient( ... 'localhost:27017', ... replicaSet='foo', ... readPreference='secondaryPreferred') >>> client.read_preference SecondaryPreferred(tag_sets=None) Now all queries will be sent to the secondary members of the set. If there are no secondary members the primary will be used as a fallback. If you have queries you would prefer to never send to the primary you can specify that using the ``secondary`` read preference. By default the read preference of a :class:`~pymongo.database.Database` is inherited from its MongoClient, and the read preference of a :class:`~pymongo.collection.Collection` is inherited from its Database. To use a different read preference use the :meth:`~pymongo.mongo_client.MongoClient.get_database` method, or the :meth:`~pymongo.database.Database.get_collection` method:: >>> from pymongo import ReadPreference >>> client.read_preference SecondaryPreferred(tag_sets=None) >>> db = client.get_database('test', read_preference=ReadPreference.SECONDARY) >>> db.read_preference Secondary(tag_sets=None) >>> coll = db.get_collection('test', read_preference=ReadPreference.PRIMARY) >>> coll.read_preference Primary() You can also change the read preference of an existing :class:`~pymongo.collection.Collection` with the :meth:`~pymongo.collection.Collection.with_options` method:: >>> coll2 = coll.with_options(read_preference=ReadPreference.NEAREST) >>> coll.read_preference Primary() >>> coll2.read_preference Nearest(tag_sets=None) Note that since most database commands can only be sent to the primary of a replica set, the :meth:`~pymongo.database.Database.command` method does not obey the Database's :attr:`~pymongo.database.Database.read_preference`, but you can pass an explicit read preference to the method:: >>> db.command('dbstats', read_preference=ReadPreference.NEAREST) {...} Reads are configured using three options: **read preference**, **tag sets**, and **local threshold**. **Read preference**: Read preference is configured using one of the classes from :mod:`~pymongo.read_preferences` (:class:`~pymongo.read_preferences.Primary`, :class:`~pymongo.read_preferences.PrimaryPreferred`, :class:`~pymongo.read_preferences.Secondary`, :class:`~pymongo.read_preferences.SecondaryPreferred`, or :class:`~pymongo.read_preferences.Nearest`). For convenience, we also provide :class:`~pymongo.read_preferences.ReadPreference` with the following attributes: - ``PRIMARY``: Read from the primary. This is the default read preference, and provides the strongest consistency. If no primary is available, raise :class:`~pymongo.errors.AutoReconnect`. - ``PRIMARY_PREFERRED``: Read from the primary if available, otherwise read from a secondary. - ``SECONDARY``: Read from a secondary. If no matching secondary is available, raise :class:`~pymongo.errors.AutoReconnect`. - ``SECONDARY_PREFERRED``: Read from a secondary if available, otherwise from the primary. - ``NEAREST``: Read from any available member. **Tag sets**: Replica-set members can be `tagged `_ according to any criteria you choose. By default, PyMongo ignores tags when choosing a member to read from, but your read preference can be configured with a ``tag_sets`` parameter. ``tag_sets`` must be a list of dictionaries, each dict providing tag values that the replica set member must match. PyMongo tries each set of tags in turn until it finds a set of tags with at least one matching member. For example, to prefer reads from the New York data center, but fall back to the San Francisco data center, tag your replica set members according to their location and create a MongoClient like so:: >>> from pymongo.read_preferences import Secondary >>> db = client.get_database( ... 'test', read_preference=Secondary([{'dc': 'ny'}, {'dc': 'sf'}])) >>> db.read_preference Secondary(tag_sets=[{'dc': 'ny'}, {'dc': 'sf'}]) MongoClient tries to find secondaries in New York, then San Francisco, and raises :class:`~pymongo.errors.AutoReconnect` if none are available. As an additional fallback, specify a final, empty tag set, ``{}``, which means "read from any member that matches the mode, ignoring tags." See :mod:`~pymongo.read_preferences` for more information. .. _distributes reads to secondaries: **Local threshold**: If multiple members match the read preference and tag sets, PyMongo reads from among the nearest members, chosen according to ping time. By default, only members whose ping times are within 15 milliseconds of the nearest are used for queries. You can choose to distribute reads among members with higher latencies by setting ``localThresholdMS`` to a larger number:: >>> client = pymongo.MongoClient( ... replicaSet='repl0', ... readPreference='secondaryPreferred', ... localThresholdMS=35) In this case, PyMongo distributes reads among matching members within 35 milliseconds of the closest member's ping time. .. note:: ``localThresholdMS`` is ignored when talking to a replica set *through* a mongos. The equivalent is the localThreshold_ command line option. .. _localThreshold: http://docs.mongodb.org/manual/reference/mongos/#cmdoption--localThreshold .. _health-monitoring: Health Monitoring ''''''''''''''''' When MongoClient is initialized it launches background threads to monitor the replica set for changes in: * Health: detect when a member goes down or comes up, or if a different member becomes primary * Configuration: detect when members are added or removed, and detect changes in members' tags * Latency: track a moving average of each member's ping time Replica-set monitoring ensures queries are continually routed to the proper members as the state of the replica set changes. .. _mongos-load-balancing: mongos Load Balancing --------------------- An instance of :class:`~pymongo.mongo_client.MongoClient` can be configured with a list of addresses of mongos servers: >>> client = MongoClient('mongodb://host1,host2,host3') Each member of the list must be a single mongos server. Multihomed and round robin DNS addresses are **not** supported. The client continuously monitors all the mongoses' availability, and its network latency to each. PyMongo distributes operations evenly among the set of mongoses within its ``localThresholdMS`` (similar to how it `distributes reads to secondaries`_ in a replica set). By default the threshold is 15 ms. The lowest-latency server, and all servers with latencies no more than ``localThresholdMS`` beyond the lowest-latency server's, receive operations equally. For example, if we have three mongoses: - host1: 20 ms - host2: 35 ms - host3: 40 ms By default the ``localThresholdMS`` is 15 ms, so PyMongo uses host1 and host2 evenly. It uses host1 because its network latency to the driver is shortest. It uses host2 because its latency is within 15 ms of the lowest-latency server's. But it excuses host3: host3 is 20ms beyond the lowest-latency server. If we set ``localThresholdMS`` to 30 ms all servers are within the threshold: >>> client = MongoClient('mongodb://host1,host2,host3/?localThresholdMS=30') .. warning:: Do **not** connect PyMongo to a pool of mongos instances through a load balancer. A single socket connection must always be routed to the same mongos instance for proper cursor support. pymongo-3.6.1/doc/examples/bulk.rst0000644000076600000240000001324213245621354017514 0ustar shanestaff00000000000000Bulk Write Operations ===================== .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('bulk_example') This tutorial explains how to take advantage of PyMongo's bulk write operation features. Executing write operations in batches reduces the number of network round trips, increasing write throughput. Bulk Insert ----------- .. versionadded:: 2.6 A batch of documents can be inserted by passing a list to the :meth:`~pymongo.collection.Collection.insert_many` method. PyMongo will automatically split the batch into smaller sub-batches based on the maximum message size accepted by MongoDB, supporting very large bulk insert operations. .. doctest:: >>> import pymongo >>> db = pymongo.MongoClient().bulk_example >>> db.test.insert_many([{'i': i} for i in range(10000)]).inserted_ids [...] >>> db.test.count() 10000 Mixed Bulk Write Operations --------------------------- .. versionadded:: 2.7 PyMongo also supports executing mixed bulk write operations. A batch of insert, update, and remove operations can be executed together using the bulk write operations API. .. _ordered_bulk: Ordered Bulk Write Operations ............................. Ordered bulk write operations are batched and sent to the server in the order provided for serial execution. The return value is an instance of :class:`~pymongo.results.BulkWriteResult` describing the type and count of operations performed. .. doctest:: :options: +NORMALIZE_WHITESPACE >>> from pprint import pprint >>> from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne >>> result = db.test.bulk_write([ ... DeleteMany({}), # Remove all documents from the previous example. ... InsertOne({'_id': 1}), ... InsertOne({'_id': 2}), ... InsertOne({'_id': 3}), ... UpdateOne({'_id': 1}, {'$set': {'foo': 'bar'}}), ... UpdateOne({'_id': 4}, {'$inc': {'j': 1}}, upsert=True), ... ReplaceOne({'j': 1}, {'j': 2})]) >>> pprint(result.bulk_api_result) {'nInserted': 3, 'nMatched': 2, 'nModified': 2, 'nRemoved': 10000, 'nUpserted': 1, 'upserted': [{u'_id': 4, u'index': 5}], 'writeConcernErrors': [], 'writeErrors': []} .. warning:: ``nModified`` is only reported by MongoDB 2.6 and later. When connected to an earlier server version, or in certain mixed version sharding configurations, PyMongo omits this field from the results of a bulk write operation. The first write failure that occurs (e.g. duplicate key error) aborts the remaining operations, and PyMongo raises :class:`~pymongo.errors.BulkWriteError`. The :attr:`details` attibute of the exception instance provides the execution results up until the failure occurred and details about the failure - including the operation that caused the failure. .. doctest:: :options: +NORMALIZE_WHITESPACE >>> from pymongo import InsertOne, DeleteOne, ReplaceOne >>> from pymongo.errors import BulkWriteError >>> requests = [ ... ReplaceOne({'j': 2}, {'i': 5}), ... InsertOne({'_id': 4}), # Violates the unique key constraint on _id. ... DeleteOne({'i': 5})] >>> try: ... db.test.bulk_write(requests) ... except BulkWriteError as bwe: ... pprint(bwe.details) ... {'nInserted': 0, 'nMatched': 1, 'nModified': 1, 'nRemoved': 0, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [{u'code': 11000, u'errmsg': u'...E11000...duplicate key error...', u'index': 1, u'op': {'_id': 4}}]} .. _unordered_bulk: Unordered Bulk Write Operations ............................... Unordered bulk write operations are batched and sent to the server in **arbitrary order** where they may be executed in parallel. Any errors that occur are reported after all operations are attempted. In the next example the first and third operations fail due to the unique constraint on _id. Since we are doing unordered execution the second and fourth operations succeed. .. doctest:: :options: +NORMALIZE_WHITESPACE >>> requests = [ ... InsertOne({'_id': 1}), ... DeleteOne({'_id': 2}), ... InsertOne({'_id': 3}), ... ReplaceOne({'_id': 4}, {'i': 1})] >>> try: ... db.test.bulk_write(requests, ordered=False) ... except BulkWriteError as bwe: ... pprint(bwe.details) ... {'nInserted': 0, 'nMatched': 1, 'nModified': 1, 'nRemoved': 1, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [{u'code': 11000, u'errmsg': u'...E11000...duplicate key error...', u'index': 0, u'op': {'_id': 1}}, {u'code': 11000, u'errmsg': u'...E11000...duplicate key error...', u'index': 2, u'op': {'_id': 3}}]} Write Concern ............. Bulk operations are executed with the :attr:`~pymongo.collection.Collection.write_concern` of the collection they are executed against. Write concern errors (e.g. wtimeout) will be reported after all operations are attempted, regardless of execution order. :: >>> from pymongo import WriteConcern >>> coll = db.get_collection( ... 'test', write_concern=WriteConcern(w=3, wtimeout=1)) >>> try: ... coll.bulk_write([InsertOne({'a': i}) for i in range(4)]) ... except BulkWriteError as bwe: ... pprint(bwe.details) ... {'nInserted': 4, 'nMatched': 0, 'nModified': 0, 'nRemoved': 0, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [{u'code': 64... u'errInfo': {u'wtimeout': True}, u'errmsg': u'waiting for replication timed out'}], 'writeErrors': []} pymongo-3.6.1/doc/examples/gevent.rst0000644000076600000240000000362613245621354020054 0ustar shanestaff00000000000000Gevent ====== PyMongo supports `Gevent `_. Simply call Gevent's ``monkey.patch_all()`` before loading any other modules: .. doctest:: >>> # You must call patch_all() *before* importing any other modules >>> from gevent import monkey >>> monkey.patch_all() >>> from pymongo import MongoClient >>> client = MongoClient() PyMongo uses thread and socket functions from the Python standard library. Gevent's monkey-patching replaces those standard functions so that PyMongo does asynchronous I/O with non-blocking sockets, and schedules operations on greenlets instead of threads. Avoid blocking in Hub.join -------------------------- By default, PyMongo uses threads to discover and monitor your servers' topology (see :ref:`health-monitoring`). If you execute ``monkey.patch_all()`` when your application first begins, PyMongo automatically uses greenlets instead of threads. When shutting down, if your application calls :meth:`~gevent.hub.Hub.join` on Gevent's :class:`~gevent.hub.Hub` without first terminating these background greenlets, the call to :meth:`~gevent.hub.Hub.join` blocks indefinitely. You therefore **must close or dereference** any active :class:`~pymongo.mongo_client.MongoClient` before exiting. An example solution to this issue in some application frameworks is a signal handler to end background greenlets when your application receives SIGHUP: .. code-block:: python import signal def graceful_reload(signum, traceback): """Explicitly close some global MongoClient object.""" client.close() signal.signal(signal.SIGHUP, graceful_reload) Applications using uWSGI prior to 1.9.16 are affected by this issue, or newer uWSGI versions with the ``-gevent-wait-for-hub`` option. See `the uWSGI changelog for details `_. pymongo-3.6.1/doc/examples/geo.rst0000644000076600000240000000532313245621354017332 0ustar shanestaff00000000000000Geospatial Indexing Example =========================== .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('geo_example') This example shows how to create and use a :data:`~pymongo.GEO2D` index in PyMongo. .. mongodoc:: geo Creating a Geospatial Index --------------------------- Creating a geospatial index in pymongo is easy: .. doctest:: >>> from pymongo import MongoClient, GEO2D >>> db = MongoClient().geo_example >>> db.places.create_index([("loc", GEO2D)]) u'loc_2d' Inserting Places ---------------- Locations in MongoDB are represented using either embedded documents or lists where the first two elements are coordinates. Here, we'll insert a couple of example locations: .. doctest:: >>> result = db.places.insert_many([{"loc": [2, 5]}, ... {"loc": [30, 5]}, ... {"loc": [1, 2]}, ... {"loc": [4, 4]}]) # doctest: +ELLIPSIS >>> result.inserted_ids [ObjectId('...'), ObjectId('...'), ObjectId('...'), ObjectId('...')] Querying -------- Using the geospatial index we can find documents near another point: .. doctest:: >>> import pprint >>> for doc in db.places.find({"loc": {"$near": [3, 6]}}).limit(3): ... pprint.pprint(doc) ... {u'_id': ObjectId('...'), u'loc': [2, 5]} {u'_id': ObjectId('...'), u'loc': [4, 4]} {u'_id': ObjectId('...'), u'loc': [1, 2]} The $maxDistance operator requires the use of :class:`~bson.son.SON`: .. doctest:: >>> from bson.son import SON >>> query = {"loc": SON([("$near", [3, 6]), ("$maxDistance", 100)])} >>> for doc in db.places.find(query).limit(3): ... pprint.pprint(doc) ... {u'_id': ObjectId('...'), u'loc': [2, 5]} {u'_id': ObjectId('...'), u'loc': [4, 4]} {u'_id': ObjectId('...'), u'loc': [1, 2]} It's also possible to query for all items within a given rectangle (specified by lower-left and upper-right coordinates): .. doctest:: >>> query = {"loc": {"$within": {"$box": [[2, 2], [5, 6]]}}} >>> for doc in db.places.find(query).sort('_id'): ... pprint.pprint(doc) {u'_id': ObjectId('...'), u'loc': [2, 5]} {u'_id': ObjectId('...'), u'loc': [4, 4]} Or circle (specified by center point and radius): .. doctest:: >>> query = {"loc": {"$within": {"$center": [[0, 0], 6]}}} >>> for doc in db.places.find(query).sort('_id'): ... pprint.pprint(doc) ... {u'_id': ObjectId('...'), u'loc': [2, 5]} {u'_id': ObjectId('...'), u'loc': [1, 2]} {u'_id': ObjectId('...'), u'loc': [4, 4]} geoNear queries are also supported using :class:`~bson.son.SON`:: >>> from bson.son import SON >>> db.command(SON([('geoNear', 'places'), ('near', [1, 2])])) {u'ok': 1.0, u'stats': ...} pymongo-3.6.1/doc/installation.rst0000644000076600000240000002146413245621354017447 0ustar shanestaff00000000000000Installing / Upgrading ====================== .. highlight:: bash **PyMongo** is in the `Python Package Index `_. .. warning:: **Do not install the "bson" package from pypi.** PyMongo comes with its own bson package; doing "pip install bson" or "easy_install bson" installs a third-party package that is incompatible with PyMongo. Installing with pip ------------------- We recommend using `pip `_ to install pymongo on all platforms:: $ python -m pip install pymongo To get a specific version of pymongo:: $ python -m pip install pymongo==3.5.1 To upgrade using pip:: $ python -m pip install --upgrade pymongo .. note:: pip does not support installing python packages in .egg format. If you would like to install PyMongo from a .egg provided on pypi use easy_install instead. Installing with easy_install ---------------------------- To use ``easy_install`` from `setuptools `_ do:: $ python -m easy_install pymongo To upgrade do:: $ python -m easy_install -U pymongo Dependencies ------------ PyMongo supports CPython 2.6, 2.7, 3.4+, PyPy, and PyPy3. Optional dependencies: GSSAPI authentication requires `pykerberos `_ on Unix or `WinKerberos `_ on Windows. The correct dependency can be installed automatically along with PyMongo:: $ python -m pip install pymongo[gssapi] Support for mongodb+srv:// URIs requires `dnspython `_:: $ python -m pip install pymongo[srv] TLS / SSL support may require `ipaddress `_ and `certifi `_ or `wincertstore `_ depending on the Python version in use. The necessary dependencies can be installed along with PyMongo:: $ python -m pip install pymongo[tls] You can install all dependencies automatically with the following command:: $ python -m pip install pymongo[gssapi,srv,tls] Other optional packages: - `backports.pbkdf2 `_, improves authentication performance with SCRAM-SHA-1, the default authentication mechanism for MongoDB 3.0+. It especially improves performance on Python versions older than 2.7.8. - `monotonic `_ adds support for a monotonic clock, which improves reliability in environments where clock adjustments are frequent. Not needed in Python 3. Installing from source ---------------------- If you'd rather install directly from the source (i.e. to stay on the bleeding edge), install the C extension dependencies then check out the latest source from github and install the driver from the resulting tree:: $ git clone git://github.com/mongodb/mongo-python-driver.git pymongo $ cd pymongo/ $ python setup.py install Installing from source on Unix .............................. To build the optional C extensions on Linux or another non-macOS Unix you must have the GNU C compiler (gcc) installed. Depending on your flavor of Unix (or Linux distribution) you may also need a python development package that provides the necessary header files for your version of Python. The package name may vary from distro to distro. Debian and Ubuntu users should issue the following command:: $ sudo apt-get install build-essential python-dev Users of Red Hat based distributions (RHEL, CentOS, Amazon Linux, Oracle Linux, Fedora, etc.) should issue the following command:: $ sudo yum install gcc python-devel Installing from source on macOS / OSX ..................................... If you want to install PyMongo with C extensions from source you will need the command line developer tools. On modern versions of macOS they can be installed by running the following in Terminal (found in /Applications/Utilities/):: xcode-select --install For older versions of OSX you may need Xcode. See the notes below for various OSX and Xcode versions. **Snow Leopard (10.6)** - Xcode 3 with 'UNIX Development Support'. **Snow Leopard Xcode 4**: The Python versions shipped with OSX 10.6.x are universal binaries. They support i386, PPC, and (in the case of python2.6) x86_64. Xcode 4 removed support for PPC, causing the distutils version shipped with Apple's builds of Python to fail to build the C extensions if you have Xcode 4 installed. There is a workaround:: # For Apple-supplied Python2.6 (installed at /usr/bin/python2.6) and # some builds from python.org $ env ARCHFLAGS='-arch i386 -arch x86_64' python -m easy_install pymongo See `http://bugs.python.org/issue11623 `_ for a more detailed explanation. **Lion (10.7) and newer** - PyMongo's C extensions can be built against versions of Python 2.7 >= 2.7.4 or Python 3.4+ downloaded from python.org. In all cases Xcode must be installed with 'UNIX Development Support'. **Xcode 5.1**: Starting with version 5.1 the version of clang that ships with Xcode throws an error when it encounters compiler flags it doesn't recognize. This may cause C extension builds to fail with an error similar to:: clang: error: unknown argument: '-mno-fused-madd' [-Wunused-command-line-argument-hard-error-in-future] There are workarounds:: # Apple specified workaround for Xcode 5.1 # easy_install $ ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future easy_install pymongo # or pip $ ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future pip install pymongo # Alternative workaround using CFLAGS # easy_install $ CFLAGS=-Qunused-arguments easy_install pymongo # or pip $ CFLAGS=-Qunused-arguments pip install pymongo Installing from source on Windows ................................. If you want to install PyMongo with C extensions from source the following requirements apply to both CPython and ActiveState's ActivePython: 64-bit Windows ~~~~~~~~~~~~~~ For Python 3.5 and newer install Visual Studio 2015. For Python 3.4 install Visual Studio 2010. You must use the full version of Visual Studio 2010 as Visual C++ Express does not provide 64-bit compilers. Make sure that you check the "x64 Compilers and Tools" option under Visual C++. For Python 2.6 and 2.7 install the `Microsoft Visual C++ Compiler for Python 2.7`_. 32-bit Windows ~~~~~~~~~~~~~~ For Python 3.5 and newer install Visual Studio 2015. For Python 3.4 install Visual C++ 2010 Express. For Python 2.6 and 2.7 install the `Microsoft Visual C++ Compiler for Python 2.7`_ .. _`Microsoft Visual C++ Compiler for Python 2.7`: https://www.microsoft.com/en-us/download/details.aspx?id=44266 .. _install-no-c: Installing Without C Extensions ------------------------------- By default, the driver attempts to build and install optional C extensions (used for increasing performance) when it is installed. If any extension fails to build the driver will be installed anyway but a warning will be printed. If you wish to install PyMongo without the C extensions, even if the extensions build properly, it can be done using a command line option to *setup.py*:: $ python setup.py --no_ext install Building PyMongo egg Packages ----------------------------- Some organizations do not allow compilers and other build tools on production systems. To install PyMongo on these systems with C extensions you may need to build custom egg packages. Make sure that you have installed the dependencies listed above for your operating system then run the following command in the PyMongo source directory:: $ python setup.py bdist_egg The egg package can be found in the dist/ subdirectory. The file name will resemble “pymongo-3.6-py2.7-linux-x86_64.egg” but may have a different name depending on your platform and the version of python you use to compile. .. warning:: These “binary distributions,” will only work on systems that resemble the environment on which you built the package. In other words, ensure that operating systems and versions of Python and architecture (i.e. “32” or “64” bit) match. Copy this file to the target system and issue the following command to install the package:: $ sudo python -m easy_install pymongo-3.6-py2.7-linux-x86_64.egg Installing a beta or release candidate -------------------------------------- MongoDB, Inc. may occasionally tag a beta or release candidate for testing by the community before final release. These releases will not be uploaded to pypi but can be found on the `github tags page `_. They can be installed by passing the full URL for the tag to pip:: $ python -m pip install https://github.com/mongodb/mongo-python-driver/archive/3.6rc0.tar.gz or easy_install:: $ python -m easy_install https://github.com/mongodb/mongo-python-driver/archive/3.6rc0.tar.gz pymongo-3.6.1/doc/contributors.rst0000644000076600000240000000441213245621354017475 0ustar shanestaff00000000000000Contributors ============ The following is a list of people who have contributed to **PyMongo**. If you belong here and are missing please let us know (or send a pull request after adding yourself to the list): - Mike Dirolf (mdirolf) - Jeff Jenkins (jeffjenkins) - Jim Jones - Eliot Horowitz (erh) - Michael Stephens (mikejs) - Joakim Sernbrant (serbaut) - Alexander Artemenko (svetlyak40wt) - Mathias Stearn (RedBeard0531) - Fajran Iman Rusadi (fajran) - Brad Clements (bkc) - Andrey Fedorov (andreyf) - Joshua Roesslein (joshthecoder) - Gregg Lind (gregglind) - Michael Schurter (schmichael) - Daniel Lundin - Michael Richardson (mtrichardson) - Dan McKinley (mcfunley) - David Wolever (wolever) - Carlos Valiente (carletes) - Jehiah Czebotar (jehiah) - Drew Perttula (drewp) - Carl Baatz (c-w-b) - Johan Bergstrom (jbergstroem) - Jonas Haag (jonashaag) - Kristina Chodorow (kchodorow) - Andrew Sibley (sibsibsib) - Flavio Percoco Premoli (FlaPer87) - Ken Kurzweil (kurzweil) - Christian Wyglendowski (dowski) - James Murty (jmurty) - Brendan W. McAdams (bwmcadams) - Bernie Hackett (behackett) - Reed O'Brien (reedobrien) - Francisco Souza (fsouza) - Alexey I. Froloff (raorn) - Steve Lacy (slacy) - Richard Shea (shearic) - Vladimir Sidorenko (gearheart) - Aaron Westendorf (awestendorf) - Dan Crosta (dcrosta) - Ryan Smith-Roberts (rmsr) - David Pisoni (gefilte) - Abhay Vardhan (abhayv) - Alexey Borzenkov (snaury) - Kostya Rybnikov (k-bx) - A Jesse Jiryu Davis (ajdavis) - Samuel Clay (samuelclay) - Ross Lawley (rozza) - Wouter Bolsterlee (wbolster) - Alex Grönholm (agronholm) - Christoph Simon (kalanzun) - Chris Tompkinson (tompko) - Mike O'Brien (mpobrien) - T Dampier (dampier) - Michael Henson (hensom) - Craig Hobbs (craigahobbs) - Emily Stolfo (estolfo) - Sam Helman (shelman) - Justin Patrin (reversefold) - Xiuming Chen (cxmcc) - Tyler Jones (thomascirca) - Amalia Hawkins (hawka) - Yuchen Ying (yegle) - Kyle Erf (3rf) - Luke Lovett (lovett89) - Jaroslav Semančík (girogiro) - Don Mitchell (dmitchell) - Ximing (armnotstrong) - Can Zhang (cannium) - Sergey Azovskov (last-g) - Heewa Barfchin (heewa) - Anna Herlihy (aherlihy) - Len Buckens (buckensl) - ultrabug - Shane Harvey (ShaneHarvey) - Cao Siyang (caosiyang) - Zhecong Kwok (gzcf) - TaoBeier(tao12345666333) - Jagrut Trivedi(Jagrut) pymongo-3.6.1/doc/api/0000755000076600000240000000000013246104133014746 5ustar shanestaff00000000000000pymongo-3.6.1/doc/api/bson/0000755000076600000240000000000013246104133015707 5ustar shanestaff00000000000000pymongo-3.6.1/doc/api/bson/objectid.rst0000644000076600000240000000127513245617773020253 0ustar shanestaff00000000000000:mod:`objectid` -- Tools for working with MongoDB ObjectIds =========================================================== .. automodule:: bson.objectid :synopsis: Tools for working with MongoDB ObjectIds .. autoclass:: bson.objectid.ObjectId(oid=None) :members: .. describe:: str(o) Get a hex encoded version of :class:`ObjectId` `o`. The following property always holds: .. testsetup:: from bson.objectid import ObjectId .. doctest:: >>> o = ObjectId() >>> o == ObjectId(str(o)) True This representation is useful for urls or other places where ``o.binary`` is inappropriate. pymongo-3.6.1/doc/api/bson/regex.rst0000644000076600000240000000040613156613521017561 0ustar shanestaff00000000000000:mod:`regex` -- Tools for representing MongoDB regular expressions ================================================================== .. versionadded:: 2.7 .. automodule:: bson.regex :synopsis: Tools for representing MongoDB regular expressions :members: pymongo-3.6.1/doc/api/bson/index.rst0000644000076600000240000000064713245617773017601 0ustar shanestaff00000000000000:mod:`bson` -- BSON (Binary JSON) Encoding and Decoding ======================================================= .. automodule:: bson :synopsis: BSON (Binary JSON) Encoding and Decoding :members: Sub-modules: .. toctree:: :maxdepth: 2 binary code codec_options dbref decimal128 errors int64 json_util max_key min_key objectid raw_bson regex son timestamp tz_util pymongo-3.6.1/doc/api/bson/codec_options.rst0000644000076600000240000000035013156613521021275 0ustar shanestaff00000000000000:mod:`codec_options` -- Tools for specifying BSON codec options =============================================================== .. automodule:: bson.codec_options :synopsis: Tools for specifying BSON codec options. :members: pymongo-3.6.1/doc/api/bson/tz_util.rst0000644000076600000240000000035213156613521020141 0ustar shanestaff00000000000000:mod:`tz_util` -- Utilities for dealing with timezones in Python ================================================================ .. automodule:: bson.tz_util :synopsis: Utilities for dealing with timezones in Python :members: pymongo-3.6.1/doc/api/bson/son.rst0000644000076600000240000000033613156613521017250 0ustar shanestaff00000000000000:mod:`son` -- Tools for working with SON, an ordered mapping ============================================================ .. automodule:: bson.son :synopsis: Tools for working with SON, an ordered mapping :members: pymongo-3.6.1/doc/api/bson/errors.rst0000644000076600000240000000033213245617773017775 0ustar shanestaff00000000000000:mod:`errors` -- Exceptions raised by the :mod:`bson` package ============================================================= .. automodule:: bson.errors :synopsis: Exceptions raised by the bson package :members: pymongo-3.6.1/doc/api/bson/min_key.rst0000644000076600000240000000037013245617773020116 0ustar shanestaff00000000000000:mod:`min_key` -- Representation for the MongoDB internal MinKey type ===================================================================== .. automodule:: bson.min_key :synopsis: Representation for the MongoDB internal MinKey type :members: pymongo-3.6.1/doc/api/bson/code.rst0000644000076600000240000000043113245617773017373 0ustar shanestaff00000000000000:mod:`code` -- Tools for representing JavaScript code ===================================================== .. automodule:: bson.code :synopsis: Tools for representing JavaScript code .. autoclass:: Code(code, scope=None, **kwargs) :members: :show-inheritance: pymongo-3.6.1/doc/api/bson/timestamp.rst0000644000076600000240000000037313245617773020471 0ustar shanestaff00000000000000:mod:`timestamp` -- Tools for representing MongoDB internal Timestamps ====================================================================== .. automodule:: bson.timestamp :synopsis: Tools for representing MongoDB internal Timestamps :members: pymongo-3.6.1/doc/api/bson/raw_bson.rst0000644000076600000240000000034013245617773020272 0ustar shanestaff00000000000000:mod:`raw_bson` -- Tools for representing raw BSON documents. ============================================================= .. automodule:: bson.raw_bson :synopsis: Tools for representing raw BSON documents. :members: pymongo-3.6.1/doc/api/bson/binary.rst0000644000076600000240000000141213245617773017745 0ustar shanestaff00000000000000:mod:`binary` -- Tools for representing binary data to be stored in MongoDB =========================================================================== .. automodule:: bson.binary :synopsis: Tools for representing binary data to be stored in MongoDB .. autodata:: BINARY_SUBTYPE .. autodata:: FUNCTION_SUBTYPE .. autodata:: OLD_BINARY_SUBTYPE .. autodata:: OLD_UUID_SUBTYPE .. autodata:: UUID_SUBTYPE .. autodata:: STANDARD .. autodata:: PYTHON_LEGACY .. autodata:: JAVA_LEGACY .. autodata:: CSHARP_LEGACY .. autodata:: MD5_SUBTYPE .. autodata:: USER_DEFINED_SUBTYPE .. autoclass:: Binary(data, subtype=BINARY_SUBTYPE) :members: :show-inheritance: .. autoclass:: UUIDLegacy(obj) :members: :show-inheritance: pymongo-3.6.1/doc/api/bson/json_util.rst0000644000076600000240000000051113245617773020466 0ustar shanestaff00000000000000:mod:`json_util` -- Tools for using Python's :mod:`json` module with BSON documents =================================================================================== .. automodule:: bson.json_util :synopsis: Tools for using Python's json module with BSON documents :members: :undoc-members: :member-order: bysource pymongo-3.6.1/doc/api/bson/dbref.rst0000644000076600000240000000046513156613521017536 0ustar shanestaff00000000000000:mod:`dbref` -- Tools for manipulating DBRefs (references to documents stored in MongoDB) ========================================================================================= .. automodule:: bson.dbref :synopsis: Tools for manipulating DBRefs (references to documents stored in MongoDB) :members: pymongo-3.6.1/doc/api/bson/decimal128.rst0000644000076600000240000000021713245617773020314 0ustar shanestaff00000000000000:mod:`decimal128` -- Support for BSON Decimal128 ================================================ .. automodule:: bson.decimal128 :members: pymongo-3.6.1/doc/api/bson/max_key.rst0000644000076600000240000000037013245617773020120 0ustar shanestaff00000000000000:mod:`max_key` -- Representation for the MongoDB internal MaxKey type ===================================================================== .. automodule:: bson.max_key :synopsis: Representation for the MongoDB internal MaxKey type :members: pymongo-3.6.1/doc/api/bson/int64.rst0000644000076600000240000000032313245617773017425 0ustar shanestaff00000000000000:mod:`int64` -- Tools for representing BSON int64 ================================================= .. versionadded:: 3.0 .. automodule:: bson.int64 :synopsis: Tools for representing BSON int64 :members: pymongo-3.6.1/doc/api/index.rst0000644000076600000240000000074513156613521016623 0ustar shanestaff00000000000000API Documentation ================= The PyMongo distribution contains three top-level packages for interacting with MongoDB. :mod:`bson` is an implementation of the `BSON format `_, :mod:`pymongo` is a full-featured driver for MongoDB, and :mod:`gridfs` is a set of tools for working with the `GridFS `_ storage specification. .. toctree:: :maxdepth: 2 bson/index pymongo/index gridfs/index pymongo-3.6.1/doc/api/pymongo/0000755000076600000240000000000013246104133016436 5ustar shanestaff00000000000000pymongo-3.6.1/doc/api/pymongo/cursor_manager.rst0000644000076600000240000000045713245617773022227 0ustar shanestaff00000000000000:mod:`cursor_manager` -- Managers to handle when cursors are killed after being closed ====================================================================================== .. automodule:: pymongo.cursor_manager :synopsis: Managers to handle when cursors are killed after being closed :members: pymongo-3.6.1/doc/api/pymongo/index.rst0000644000076600000240000000206413245621354020311 0ustar shanestaff00000000000000:mod:`pymongo` -- Python driver for MongoDB =========================================== .. automodule:: pymongo :synopsis: Python driver for MongoDB .. autodata:: version .. data:: MongoClient Alias for :class:`pymongo.mongo_client.MongoClient`. .. data:: MongoReplicaSetClient Alias for :class:`pymongo.mongo_replica_set_client.MongoReplicaSetClient`. .. data:: ReadPreference Alias for :class:`pymongo.read_preferences.ReadPreference`. .. autofunction:: has_c .. data:: MIN_SUPPORTED_WIRE_VERSION The minimum wire protocol version PyMongo supports. .. data:: MAX_SUPPORTED_WIRE_VERSION The maximum wire protocol version PyMongo supports. Sub-modules: .. toctree:: :maxdepth: 2 database change_stream client_session collation collection command_cursor cursor bulk errors message monitoring mongo_client mongo_replica_set_client operations pool read_concern read_preferences results son_manipulator cursor_manager uri_parser write_concern pymongo-3.6.1/doc/api/pymongo/server_description.rst0000644000076600000240000000071013245617773023121 0ustar shanestaff00000000000000:orphan: :mod:`server_description` -- An object representation of a server the driver is connected to. ============================================================================================= .. automodule:: pymongo.server_description .. autoclass:: pymongo.server_description.ServerDescription() .. autoattribute:: address .. autoattribute:: all_hosts .. autoattribute:: server_type .. autoattribute:: server_type_name pymongo-3.6.1/doc/api/pymongo/results.rst0000644000076600000240000000030213156613521020672 0ustar shanestaff00000000000000:mod:`results` -- Result class definitions ========================================== .. automodule:: pymongo.results :synopsis: Result class definitions :members: :inherited-members: pymongo-3.6.1/doc/api/pymongo/mongo_client.rst0000644000076600000240000000352513245621354021662 0ustar shanestaff00000000000000:mod:`mongo_client` -- Tools for connecting to MongoDB ====================================================== .. automodule:: pymongo.mongo_client :synopsis: Tools for connecting to MongoDB .. autoclass:: pymongo.mongo_client.MongoClient(host='localhost', port=27017, document_class=dict, tz_aware=False, connect=True, **kwargs) .. automethod:: close .. describe:: c[db_name] || c.db_name Get the `db_name` :class:`~pymongo.database.Database` on :class:`MongoClient` `c`. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. .. autoattribute:: event_listeners .. autoattribute:: address .. autoattribute:: primary .. autoattribute:: secondaries .. autoattribute:: arbiters .. autoattribute:: is_primary .. autoattribute:: is_mongos .. autoattribute:: max_pool_size .. autoattribute:: min_pool_size .. autoattribute:: max_idle_time_ms .. autoattribute:: nodes .. autoattribute:: max_bson_size .. autoattribute:: max_message_size .. autoattribute:: max_write_batch_size .. autoattribute:: local_threshold_ms .. autoattribute:: server_selection_timeout .. autoattribute:: codec_options .. autoattribute:: read_preference .. autoattribute:: write_concern .. autoattribute:: read_concern .. autoattribute:: is_locked .. automethod:: start_session .. automethod:: list_databases .. automethod:: list_database_names .. automethod:: database_names .. automethod:: drop_database .. automethod:: get_database .. automethod:: server_info .. automethod:: close_cursor .. automethod:: kill_cursors .. automethod:: set_cursor_manager .. automethod:: fsync .. automethod:: unlock .. automethod:: get_default_database pymongo-3.6.1/doc/api/pymongo/change_stream.rst0000644000076600000240000000024013245621354021774 0ustar shanestaff00000000000000:mod:`change_stream` -- Watch changes on a collection ===================================================== .. automodule:: pymongo.change_stream :members: pymongo-3.6.1/doc/api/pymongo/errors.rst0000644000076600000240000000034612727574422020526 0ustar shanestaff00000000000000:mod:`errors` -- Exceptions raised by the :mod:`pymongo` package ================================================================ .. automodule:: pymongo.errors :synopsis: Exceptions raised by the pymongo package :members: pymongo-3.6.1/doc/api/pymongo/cursor.rst0000644000076600000240000000264513245621354020524 0ustar shanestaff00000000000000:mod:`cursor` -- Tools for iterating over MongoDB query results =============================================================== .. automodule:: pymongo.cursor :synopsis: Tools for iterating over MongoDB query results .. autoclass:: pymongo.cursor.CursorType .. autoattribute:: NON_TAILABLE :annotation: .. autoattribute:: TAILABLE :annotation: .. autoattribute:: TAILABLE_AWAIT :annotation: .. autoattribute:: EXHAUST :annotation: .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None) :members: .. describe:: c[index] See :meth:`__getitem__`. .. automethod:: __getitem__ .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None) pymongo-3.6.1/doc/api/pymongo/write_concern.rst0000644000076600000240000000033413156613521022037 0ustar shanestaff00000000000000:mod:`write_concern` -- Tools for specifying write concern ========================================================== .. automodule:: pymongo.write_concern :synopsis: Tools for specifying write concern. :members: pymongo-3.6.1/doc/api/pymongo/read_preferences.rst0000644000076600000240000000214113245621354022472 0ustar shanestaff00000000000000:mod:`read_preferences` -- Utilities for choosing which member of a replica set to read from. ============================================================================================= .. automodule:: pymongo.read_preferences :synopsis: Utilities for choosing which member of a replica set to read from. .. autoclass:: pymongo.read_preferences.Primary .. max_staleness, min_wire_version, mongos_mode, and tag_sets don't make sense for Primary. .. autoattribute:: document .. autoattribute:: mode .. autoattribute:: name .. autoclass:: pymongo.read_preferences.PrimaryPreferred :inherited-members: .. autoclass:: pymongo.read_preferences.Secondary :inherited-members: .. autoclass:: pymongo.read_preferences.SecondaryPreferred :inherited-members: .. autoclass:: pymongo.read_preferences.Nearest :inherited-members: .. autoclass:: ReadPreference .. autoattribute:: PRIMARY .. autoattribute:: PRIMARY_PREFERRED .. autoattribute:: SECONDARY .. autoattribute:: SECONDARY_PREFERRED .. autoattribute:: NEAREST pymongo-3.6.1/doc/api/pymongo/command_cursor.rst0000644000076600000240000000041013245621354022206 0ustar shanestaff00000000000000:mod:`command_cursor` -- Tools for iterating over MongoDB command results ========================================================================= .. automodule:: pymongo.command_cursor :synopsis: Tools for iterating over MongoDB command results :members: pymongo-3.6.1/doc/api/pymongo/pool.rst0000644000076600000240000000033513156613521020150 0ustar shanestaff00000000000000:mod:`pool` -- Pool module for use with a MongoDB client. ============================================================== .. automodule:: pymongo.pool :synopsis: Pool module for use with a MongoDB client. :members: pymongo-3.6.1/doc/api/pymongo/monitoring.rst0000644000076600000240000000302613245617773021400 0ustar shanestaff00000000000000:mod:`monitoring` -- Tools for monitoring driver events. ======================================================== .. automodule:: pymongo.monitoring :synopsis: Tools for monitoring driver events. .. autofunction:: register(listener) .. autoclass:: CommandListener :members: :inherited-members: .. autoclass:: ServerListener :members: :inherited-members: .. autoclass:: ServerHeartbeatListener :members: :inherited-members: .. autoclass:: TopologyListener :members: :inherited-members: .. autoclass:: CommandStartedEvent :members: :inherited-members: .. autoclass:: CommandSucceededEvent :members: :inherited-members: .. autoclass:: CommandFailedEvent :members: :inherited-members: .. autoclass:: ServerDescriptionChangedEvent :members: :inherited-members: .. autoclass:: ServerOpeningEvent :members: :inherited-members: .. autoclass:: ServerClosedEvent :members: :inherited-members: .. autoclass:: TopologyDescriptionChangedEvent :members: :inherited-members: .. autoclass:: TopologyOpenedEvent :members: :inherited-members: .. autoclass:: TopologyClosedEvent :members: :inherited-members: .. autoclass:: ServerHeartbeatStartedEvent :members: :inherited-members: .. autoclass:: ServerHeartbeatSucceededEvent :members: :inherited-members: .. autoclass:: ServerHeartbeatFailedEvent :members: :inherited-members: pymongo-3.6.1/doc/api/pymongo/database.rst0000644000076600000240000000176513245621354020755 0ustar shanestaff00000000000000:mod:`database` -- Database level operations ============================================ .. automodule:: pymongo.database :synopsis: Database level operations .. autodata:: pymongo.auth.MECHANISMS .. autodata:: pymongo.OFF .. autodata:: pymongo.SLOW_ONLY .. autodata:: pymongo.ALL .. autoclass:: pymongo.database.Database :members: .. describe:: db[collection_name] || db.collection_name Get the `collection_name` :class:`~pymongo.collection.Collection` of :class:`Database` `db`. Raises :class:`~pymongo.errors.InvalidName` if an invalid collection name is used. .. note:: Use dictionary style access if `collection_name` is an attribute of the :class:`Database` class eg: db[`collection_name`]. .. autoattribute:: codec_options .. autoattribute:: read_preference .. autoattribute:: write_concern .. autoattribute:: read_concern .. autoclass:: pymongo.database.SystemJS :members: pymongo-3.6.1/doc/api/pymongo/operations.rst0000644000076600000240000000027513156613521021365 0ustar shanestaff00000000000000:mod:`operations` -- Operation class definitions ================================================ .. automodule:: pymongo.operations :synopsis: Operation class definitions :members: pymongo-3.6.1/doc/api/pymongo/uri_parser.rst0000644000076600000240000000035013156613521021347 0ustar shanestaff00000000000000:mod:`uri_parser` -- Tools to parse and validate a MongoDB URI ============================================================== .. automodule:: pymongo.uri_parser :synopsis: Tools to parse and validate a MongoDB URI. :members: pymongo-3.6.1/doc/api/pymongo/bulk.rst0000644000076600000240000000030413156613521020130 0ustar shanestaff00000000000000:mod:`bulk` -- The bulk write operations interface ================================================== .. automodule:: pymongo.bulk :synopsis: The bulk write operations interface. :members: pymongo-3.6.1/doc/api/pymongo/ismaster.rst0000644000076600000240000000037313245617773021044 0ustar shanestaff00000000000000:orphan: :mod:`ismaster` -- A wrapper for ismaster command responses. ============================================================ .. automodule:: pymongo.ismaster .. autoclass:: pymongo.ismaster.IsMaster(doc) .. autoattribute:: document pymongo-3.6.1/doc/api/pymongo/mongo_replica_set_client.rst0000644000076600000240000000247313245617773024247 0ustar shanestaff00000000000000:mod:`mongo_replica_set_client` -- Tools for connecting to a MongoDB replica set ================================================================================ .. automodule:: pymongo.mongo_replica_set_client :synopsis: Tools for connecting to a MongoDB replica set .. autoclass:: pymongo.mongo_replica_set_client.MongoReplicaSetClient(hosts_or_uri, document_class=dict, tz_aware=False, connect=True, **kwargs) .. automethod:: close .. describe:: c[db_name] || c.db_name Get the `db_name` :class:`~pymongo.database.Database` on :class:`MongoReplicaSetClient` `c`. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. .. autoattribute:: primary .. autoattribute:: secondaries .. autoattribute:: arbiters .. autoattribute:: max_pool_size .. autoattribute:: max_bson_size .. autoattribute:: max_message_size .. autoattribute:: local_threshold_ms .. autoattribute:: codec_options .. autoattribute:: read_preference .. autoattribute:: write_concern .. automethod:: database_names .. automethod:: drop_database .. automethod:: get_database .. automethod:: close_cursor .. automethod:: kill_cursors .. automethod:: set_cursor_manager .. automethod:: get_default_database pymongo-3.6.1/doc/api/pymongo/son_manipulator.rst0000644000076600000240000000052012727574422022416 0ustar shanestaff00000000000000:mod:`son_manipulator` -- Manipulators that can edit SON documents as they are saved or retrieved ================================================================================================= .. automodule:: pymongo.son_manipulator :synopsis: Manipulators that can edit SON documents as they are saved or retrieved :members: pymongo-3.6.1/doc/api/pymongo/topology_description.rst0000644000076600000240000000106013245617773023466 0ustar shanestaff00000000000000:orphan: :mod:`topology_description` -- An object representation of a deployment of MongoDB servers. =========================================================================================== .. automodule:: pymongo.topology_description .. autoclass:: pymongo.topology_description.TopologyDescription() .. automethod:: has_readable_server(read_preference=ReadPreference.PRIMARY) .. automethod:: has_writable_server .. automethod:: server_descriptions .. autoattribute:: topology_type .. autoattribute:: topology_type_name pymongo-3.6.1/doc/api/pymongo/message.rst0000644000076600000240000000036612727574422020640 0ustar shanestaff00000000000000:mod:`message` -- Tools for creating messages to be sent to MongoDB =================================================================== .. automodule:: pymongo.message :synopsis: Tools for creating messages to be sent to MongoDB :members: pymongo-3.6.1/doc/api/pymongo/collection.rst0000644000076600000240000000741313245621354021340 0ustar shanestaff00000000000000:mod:`collection` -- Collection level operations ================================================ .. automodule:: pymongo.collection :synopsis: Collection level operations .. autodata:: pymongo.ASCENDING .. autodata:: pymongo.DESCENDING .. autodata:: pymongo.GEO2D .. autodata:: pymongo.GEOHAYSTACK .. autodata:: pymongo.GEOSPHERE .. autodata:: pymongo.HASHED .. autodata:: pymongo.TEXT .. autoclass:: pymongo.collection.ReturnDocument .. autoattribute:: BEFORE :annotation: .. autoattribute:: AFTER :annotation: .. autoclass:: pymongo.collection.Collection(database, name, create=False, **kwargs) .. describe:: c[name] || c.name Get the `name` sub-collection of :class:`Collection` `c`. Raises :class:`~pymongo.errors.InvalidName` if an invalid collection name is used. .. autoattribute:: full_name .. autoattribute:: name .. autoattribute:: database .. autoattribute:: codec_options .. autoattribute:: read_preference .. autoattribute:: write_concern .. autoattribute:: read_concern .. automethod:: with_options .. automethod:: bulk_write .. automethod:: insert_one .. automethod:: insert_many .. automethod:: replace_one .. automethod:: update_one .. automethod:: update_many .. automethod:: delete_one .. automethod:: delete_many .. automethod:: aggregate .. automethod:: aggregate_raw_batches .. automethod:: watch .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None) .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None) .. automethod:: find_one(filter=None, *args, **kwargs) .. automethod:: find_one_and_delete .. automethod:: find_one_and_replace(filter, replacement, projection=None, sort=None, return_document=ReturnDocument.BEFORE, session=None, **kwargs) .. automethod:: find_one_and_update(filter, update, projection=None, sort=None, return_document=ReturnDocument.BEFORE, array_filters=None, session=None, **kwargs) .. automethod:: count .. automethod:: distinct .. automethod:: create_index .. automethod:: create_indexes .. automethod:: drop_index .. automethod:: drop_indexes .. automethod:: reindex .. automethod:: list_indexes .. automethod:: index_information .. automethod:: drop .. automethod:: rename .. automethod:: options .. automethod:: map_reduce .. automethod:: inline_map_reduce .. automethod:: parallel_scan .. automethod:: initialize_unordered_bulk_op .. automethod:: initialize_ordered_bulk_op .. automethod:: group .. automethod:: insert(doc_or_docs, manipulate=True, check_keys=True, continue_on_error=False, **kwargs) .. automethod:: save(to_save, manipulate=True, check_keys=True, **kwargs) .. automethod:: update(spec, document, upsert=False, manipulate=False, multi=False, check_keys=True, **kwargs) .. automethod:: remove(spec_or_id=None, multi=True, **kwargs) .. automethod:: find_and_modify .. automethod:: ensure_index pymongo-3.6.1/doc/api/pymongo/collation.rst0000644000076600000240000000120613245617773021175 0ustar shanestaff00000000000000:mod:`collation` -- Tools for working with collations. ====================================================== .. automodule:: pymongo.collation :synopsis: Tools for working with collations. .. autoclass:: pymongo.collation.Collation .. autoclass:: pymongo.collation.CollationStrength :members: :member-order: bysource .. autoclass:: pymongo.collation.CollationAlternate :members: :member-order: bysource .. autoclass:: pymongo.collation.CollationCaseFirst :members: :member-order: bysource .. autoclass:: pymongo.collation.CollationMaxVariable :members: :member-order: bysource pymongo-3.6.1/doc/api/pymongo/read_concern.rst0000644000076600000240000000036513245617773021640 0ustar shanestaff00000000000000:mod:`read_concern` -- Tools for working with read concern. =========================================================== .. automodule:: pymongo.read_concern :synopsis: Tools for working with read concern. :members: :inherited-members: pymongo-3.6.1/doc/api/pymongo/client_session.rst0000644000076600000240000000027513245621354022225 0ustar shanestaff00000000000000:mod:`client_session` -- Logical sessions for sequential operations =================================================================== .. automodule:: pymongo.client_session :members: pymongo-3.6.1/doc/api/gridfs/0000755000076600000240000000000013246104133016224 5ustar shanestaff00000000000000pymongo-3.6.1/doc/api/gridfs/index.rst0000644000076600000240000000036312727574422020106 0ustar shanestaff00000000000000:mod:`gridfs` -- Tools for working with GridFS ============================================== .. automodule:: gridfs :synopsis: Tools for working with GridFS :members: Sub-modules: .. toctree:: :maxdepth: 2 errors grid_file pymongo-3.6.1/doc/api/gridfs/errors.rst0000644000076600000240000000034412727574422020312 0ustar shanestaff00000000000000:mod:`errors` -- Exceptions raised by the :mod:`gridfs` package ================================================================= .. automodule:: gridfs.errors :synopsis: Exceptions raised by the gridfs package :members: pymongo-3.6.1/doc/api/gridfs/grid_file.rst0000644000076600000240000000070313245617773020724 0ustar shanestaff00000000000000:mod:`grid_file` -- Tools for representing files stored in GridFS ================================================================= .. automodule:: gridfs.grid_file :synopsis: Tools for representing files stored in GridFS .. autoclass:: GridIn :members: .. autoattribute:: _id .. autoclass:: GridOut :members: .. autoattribute:: _id .. automethod:: __iter__ .. autoclass:: GridOutCursor :members: pymongo-3.6.1/doc/changelog.rst0000644000076600000240000030304413246100114016655 0ustar shanestaff00000000000000Changelog ========= Changes in Version 3.6.1 ------------------------ Version 3.6.1 fixes bugs reported since the release of 3.6.0: - Fix regression in PyMongo 3.5.0 that causes idle sockets to be closed almost instantly when ``maxIdleTimeMS`` is set. Idle sockets are now closed after ``maxIdleTimeMS`` milliseconds. - :attr:`pymongo.mongo_client.MongoClient.max_idle_time_ms` now returns milliseconds instead of seconds. - Properly import and use the `monotonic `_ library for monotonic time when it is installed. - :meth:`~pymongo.collection.Collection.aggregate` now ignores the ``batchSize`` argument when running a pipeline with a ``$out`` stage. - Always send handshake metadata for new connections. Issues Resolved ............... See the `PyMongo 3.6.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.6.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=19438 Changes in Version 3.6.0 ------------------------ Version 3.6 adds support for MongoDB 3.6, drops support for CPython 3.3 (PyPy3 is still supported), and drops support for MongoDB versions older than 2.6. If connecting to a MongoDB 2.4 server or older, PyMongo now throws a :exc:`~pymongo.errors.ConfigurationError`. Highlights include: - Support for change streams. See the :meth:`~pymongo.collection.Collection.watch` method for details. - Support for array_filters in :meth:`~pymongo.collection.Collection.update_one`, :meth:`~pymongo.collection.Collection.update_many`, :meth:`~pymongo.collection.Collection.find_one_and_update`, :meth:`~pymongo.operations.UpdateOne`, and :meth:`~pymongo.operations.UpdateMany`. - New Session API, see :meth:`~pymongo.mongo_client.MongoClient.start_session`. - New methods :meth:`~pymongo.collection.Collection.find_raw_batches` and :meth:`~pymongo.collection.Collection.aggregate_raw_batches` for use with external libraries that can parse raw batches of BSON data. - New methods :meth:`~pymongo.mongo_client.MongoClient.list_databases` and :meth:`~pymongo.mongo_client.MongoClient.list_database_names`. - New methods :meth:`~pymongo.database.Database.list_collections` and :meth:`~pymongo.database.Database.list_collection_names`. - Support for mongodb+srv:// URIs. See :class:`~pymongo.mongo_client.MongoClient` for details. - Index management helpers (:meth:`~pymongo.collection.Collection.create_index`, :meth:`~pymongo.collection.Collection.create_indexes`, :meth:`~pymongo.collection.Collection.drop_index`, :meth:`~pymongo.collection.Collection.drop_indexes`, :meth:`~pymongo.collection.Collection.reindex`) now support maxTimeMS. - Support for retryable writes and the ``retryWrites`` URI option. See :class:`~pymongo.mongo_client.MongoClient` for details. Deprecations: - The `useCursor` option for :meth:`~pymongo.collection.Collection.aggregate` is deprecated. The option was only necessary when upgrading from MongoDB 2.4 to MongoDB 2.6. MongoDB 2.4 is no longer supported. - The :meth:`~pymongo.database.Database.add_user` and :meth:`~pymongo.database.Database.remove_user` methods are deprecated. See the method docstrings for alternatives. Unavoidable breaking changes: - Starting in MongoDB 3.6, the deprecated methods :meth:`~pymongo.database.Database.authenticate` and :meth:`~pymongo.database.Database.logout` now invalidate all cursors created prior. Instead of using these methods to change credentials, pass credentials for one user to the :class:`~pymongo.mongo_client.MongoClient` at construction time, and either grant access to several databases to one user account, or use a distinct client object for each user. - BSON binary subtype 4 is decoded using RFC-4122 byte order regardless of the UUID representation. This is a change in behavior for applications that use UUID representation :data:`bson.binary.JAVA_LEGACY` or :data:`bson.binary.CSHARP_LEGACY` to decode BSON binary subtype 4. Other UUID representations, :data:`bson.binary.PYTHON_LEGACY` (the default) and :data:`bson.binary.STANDARD`, and the decoding of BSON binary subtype 3 are unchanged. Issues Resolved ............... See the `PyMongo 3.6 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.6 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=18043 Changes in Version 3.5.1 ------------------------ Version 3.5.1 fixes bugs reported since the release of 3.5.0: - Work around socket.getsockopt issue with NetBSD. - :meth:`pymongo.command_cursor.CommandCursor.close` now closes the cursor synchronously instead of deferring to a background thread. - Fix documentation build warnings with Sphinx 1.6.x. Issues Resolved ............... See the `PyMongo 3.5.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.5.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=18721 Changes in Version 3.5 ---------------------- Version 3.5 implements a number of improvements and bug fixes: Highlights include: - Username and password can be passed to :class:`~pymongo.mongo_client.MongoClient` as keyword arguments. Before, the only way to pass them was in the URI. - Increased the performance of using :class:`~bson.raw_bson.RawBSONDocument`. - Increased the performance of :meth:`~pymongo.mongo_client.MongoClient.database_names` by using the `nameOnly` option for listDatabases when available. - Increased the performance of :meth:`~pymongo.collection.Collection.bulk_write` by reducing the memory overhead of :class:`~pymongo.operations.InsertOne`, :class:`~pymongo.operations.DeleteOne`, and :class:`~pymongo.operations.DeleteMany`. - Added the `collation` option to :class:`~pymongo.operations.DeleteOne`, :class:`~pymongo.operations.DeleteMany`, :class:`~pymongo.operations.ReplaceOne`, :class:`~pymongo.operations.UpdateOne`, and :class:`~pymongo.operations.UpdateMany`. - Implemented the `MongoDB Extended JSON `_ specification. - :class:`~bson.decimal128.Decimal128` now works when cdecimal is installed. - PyMongo is now tested against a wider array of operating systems and CPU architectures (including s390x, ARM64, and POWER8). Changes and Deprecations: - :meth:`~pymongo.collection.Collection.find` has new options `return_key`, `show_record_id`, `snapshot`, `hint`, `max_time_ms`, `max_scan`, `min`, `max`, and `comment`. Deprecated the option `modifiers`. - Deprecated :meth:`~pymongo.collection.Collection.group`. The group command was deprecated in MongoDB 3.4 and is expected to be removed in MongoDB 3.6. Applications should use :meth:`~pymongo.collection.Collection.aggregate` with the `$group` pipeline stage instead. - Deprecated :meth:`~pymongo.database.Database.authenticate`. Authenticating multiple users conflicts with support for logical sessions in MongoDB 3.6. To authenticate as multiple users, create multiple instances of :class:`~pymongo.mongo_client.MongoClient`. - Deprecated :meth:`~pymongo.database.Database.eval`. The eval command was deprecated in MongoDB 3.0 and will be removed in a future server version. - Deprecated :class:`~pymongo.database.SystemJS`. - Deprecated :meth:`~pymongo.mongo_client.MongoClient.get_default_database`. Applications should use :meth:`~pymongo.mongo_client.MongoClient.get_database` without the `name` parameter instead. - Deprecated the MongoClient option `socketKeepAlive`. It now defaults to true and disabling it is not recommended, see `does TCP keepalive time affect MongoDB Deployments? `_ - Deprecated :meth:`~pymongo.collection.Collection.initialize_ordered_bulk_op`, :meth:`~pymongo.collection.Collection.initialize_unordered_bulk_op`, and :class:`~pymongo.bulk.BulkOperationBuilder`. Use :meth:`~pymongo.collection.Collection.bulk_write` instead. - Deprecated :const:`~bson.json_util.STRICT_JSON_OPTIONS`. Use :const:`~bson.json_util.RELAXED_JSON_OPTIONS` or :const:`~bson.json_util.CANONICAL_JSON_OPTIONS` instead. - If a custom :class:`~bson.codec_options.CodecOptions` is passed to :class:`RawBSONDocument`, its `document_class` must be :class:`RawBSONDocument`. - :meth:`~pymongo.collection.Collection.list_indexes` no longer raises OperationFailure when the collection (or database) does not exist on MongoDB >= 3.0. Instead, it returns an empty :class:`~pymongo.command_cursor.CommandCursor` to make the behavior consistent across all MongoDB versions. - In Python 3, :meth:`~bson.json_util.loads` now automatically decodes JSON $binary with a subtype of 0 into :class:`bytes` instead of :class:`~bson.binary.Binary`. See the :doc:`/python3` for more details. - :meth:`~bson.json_util.loads` now raises ``TypeError`` or ``ValueError`` when parsing JSON type wrappers with values of the wrong type or any extra keys. - :meth:`pymongo.cursor.Cursor.close` and :meth:`pymongo.mongo_client.MongoClient.close` now kill cursors synchronously instead of deferring to a background thread. - :meth:`~pymongo.uri_parser.parse_uri` now returns the original value of the ``readPreference`` MongoDB URI option instead of the validated read preference mode. Issues Resolved ............... See the `PyMongo 3.5 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.5 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=17590 Changes in Version 3.4 ---------------------- Version 3.4 implements the new server features introduced in MongoDB 3.4 and a whole lot more: Highlights include: - Complete support for MongoDB 3.4: - Unicode aware string comparison using :doc:`examples/collations`. - Support for the new :class:`~bson.decimal128.Decimal128` BSON type. - A new maxStalenessSeconds read preference option. - A username is no longer required for the MONGODB-X509 authentication mechanism when connected to MongoDB >= 3.4. - :meth:`~pymongo.collection.Collection.parallel_scan` supports maxTimeMS. - :attr:`~pymongo.write_concern.WriteConcern` is automatically applied by all helpers for commands that write to the database when connected to MongoDB 3.4+. This change affects the following helpers: - :meth:`~pymongo.mongo_client.MongoClient.drop_database` - :meth:`~pymongo.database.Database.create_collection` - :meth:`~pymongo.database.Database.drop_collection` - :meth:`~pymongo.collection.Collection.aggregate` (when using $out) - :meth:`~pymongo.collection.Collection.create_indexes` - :meth:`~pymongo.collection.Collection.create_index` - :meth:`~pymongo.collection.Collection.drop_indexes` - :meth:`~pymongo.collection.Collection.drop_indexes` - :meth:`~pymongo.collection.Collection.drop_index` - :meth:`~pymongo.collection.Collection.map_reduce` (when output is not "inline") - :meth:`~pymongo.collection.Collection.reindex` - :meth:`~pymongo.collection.Collection.rename` - Improved support for logging server discovery and monitoring events. See :mod:`~pymongo.monitoring` for examples. - Support for matching iPAddress subjectAltName values for TLS certificate verification. - TLS compression is now explicitly disabled when possible. - The Server Name Indication (SNI) TLS extension is used when possible. - Finer control over JSON encoding/decoding with :class:`~bson.json_util.JSONOptions`. - Allow :class:`~bson.code.Code` objects to have a scope of ``None``, signifying no scope. Also allow encoding Code objects with an empty scope (i.e. ``{}``). .. warning:: Starting in PyMongo 3.4, :attr:`bson.code.Code.scope` may return ``None``, as the default scope is ``None`` instead of ``{}``. .. note:: PyMongo 3.4+ attempts to create sockets non-inheritable when possible (i.e. it sets the close-on-exec flag on socket file descriptors). Support is limited to a subset of POSIX operating systems (not including Windows) and the flag usually cannot be set in a single atomic operation. CPython 3.4+ implements `PEP 446`_, creating all file descriptors non-inheritable by default. Users that require this behavior are encouraged to upgrade to CPython 3.4+. Since 3.4rc0, the max staleness option has been renamed from ``maxStalenessMS`` to ``maxStalenessSeconds``, its smallest value has changed from twice ``heartbeatFrequencyMS`` to 90 seconds, and its default value has changed from ``None`` or 0 to -1. .. _PEP 446: https://www.python.org/dev/peps/pep-0446/ Issues Resolved ............... See the `PyMongo 3.4 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.4 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16594 Changes in Version 3.3.1 ------------------------ Version 3.3.1 fixes a memory leak when decoding elements inside of a :class:`~bson.raw_bson.RawBSONDocument`. Issues Resolved ............... See the `PyMongo 3.3.1 release notes in Jira`_ for the list of resolved issues in this release. .. _PyMongo 3.3.1 release notes in Jira: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=17636 Changes in Version 3.3 ---------------------- Version 3.3 adds the following major new features: - C extensions support on big endian systems. - Kerberos authentication support on Windows using `WinKerberos `_. - A new ``ssl_clrfile`` option to support certificate revocation lists. - A new ``ssl_pem_passphrase`` option to support encrypted key files. - Support for publishing server discovery and monitoring events. See :mod:`~pymongo.monitoring` for details. - New connection pool options ``minPoolSize`` and ``maxIdleTimeMS``. - New ``heartbeatFrequencyMS`` option controls the rate at which background monitoring threads re-check servers. Default is once every 10 seconds. .. warning:: PyMongo 3.3 drops support for MongoDB versions older than 2.4. It also drops support for python 3.2 (pypy3 continues to be supported). Issues Resolved ............... See the `PyMongo 3.3 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16005 Changes in Version 3.2.2 ------------------------ Version 3.2.2 fixes a few issues reported since the release of 3.2.1, including a fix for using the `connect` option in the MongoDB URI and support for setting the batch size for a query to 1 when using MongoDB 3.2+. Issues Resolved ............... See the `PyMongo 3.2.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.2.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16538 Changes in Version 3.2.1 ------------------------ Version 3.2.1 fixes a few issues reported since the release of 3.2, including running the mapreduce command twice when calling the :meth:`~pymongo.collection.Collection.inline_map_reduce` method and a :exc:`TypeError` being raised when calling :meth:`~gridfs.GridFSBucket.download_to_stream`. This release also improves error messaging around BSON decoding. Issues Resolved ............... See the `PyMongo 3.2.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.2.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16312 Changes in Version 3.2 ---------------------- Version 3.2 implements the new server features introduced in MongoDB 3.2. Highlights include: - Full support for MongoDB 3.2 including: - Support for :class:`~pymongo.read_concern.ReadConcern` - :class:`~pymongo.write_concern.WriteConcern` is now applied to :meth:`~pymongo.collection.Collection.find_one_and_replace`, :meth:`~pymongo.collection.Collection.find_one_and_update`, and :meth:`~pymongo.collection.Collection.find_one_and_delete`. - Support for the new `bypassDocumentValidation` option in write helpers. - Support for reading and writing raw BSON with :class:`~bson.raw_bson.RawBSONDocument` .. note:: Certain :class:`~pymongo.mongo_client.MongoClient` properties now block until a connection is established or raise :exc:`~pymongo.errors.ServerSelectionTimeoutError` if no server is available. See :class:`~pymongo.mongo_client.MongoClient` for details. Issues Resolved ............... See the `PyMongo 3.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=15612 Changes in Version 3.1.1 ------------------------ Version 3.1.1 fixes a few issues reported since the release of 3.1, including a regression in error handling for oversize command documents and interrupt handling issues in the C extensions. Issues Resolved ............... See the `PyMongo 3.1.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.1.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16211 Changes in Version 3.1 ---------------------- Version 3.1 implements a few new features and fixes bugs reported since the release of 3.0.3. Highlights include: - Command monitoring support. See :mod:`~pymongo.monitoring` for details. - Configurable error handling for :exc:`UnicodeDecodeError`. See the `unicode_decode_error_handler` option of :class:`~bson.codec_options.CodecOptions`. - Optional automatic timezone conversion when decoding BSON datetime. See the `tzinfo` option of :class:`~bson.codec_options.CodecOptions`. - An implementation of :class:`~gridfs.GridFSBucket` from the new GridFS spec. - Compliance with the new Connection String spec. - Reduced idle CPU usage in Python 2. Changes in internal classes ........................... The private ``PeriodicExecutor`` class no longer takes a ``condition_class`` option, and the private ``thread_util.Event`` class is removed. Issues Resolved ............... See the `PyMongo 3.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=14796 Changes in Version 3.0.3 ------------------------ Version 3.0.3 fixes issues reported since the release of 3.0.2, including a feature breaking bug in the GSSAPI implementation. Issues Resolved ............... See the `PyMongo 3.0.3 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.0.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=15528 Changes in Version 3.0.2 ------------------------ Version 3.0.2 fixes issues reported since the release of 3.0.1, most importantly a bug that could route operations to replica set members that are not in primary or secondary state when using :class:`~pymongo.read_preferences.PrimaryPreferred` or :class:`~pymongo.read_preferences.Nearest`. It is a recommended upgrade for all users of PyMongo 3.0.x. Issues Resolved ............... See the `PyMongo 3.0.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.0.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=15430 Changes in Version 3.0.1 ------------------------ Version 3.0.1 fixes issues reported since the release of 3.0, most importantly a bug in GridFS.delete that could prevent file chunks from actually being deleted. Issues Resolved ............... See the `PyMongo 3.0.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.0.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=15322 Changes in Version 3.0 ---------------------- PyMongo 3.0 is a partial rewrite of PyMongo bringing a large number of improvements: - A unified client class. MongoClient is the one and only client class for connecting to a standalone mongod, replica set, or sharded cluster. Migrating from a standalone, to a replica set, to a sharded cluster can be accomplished with only a simple URI change. - MongoClient is much more responsive to configuration changes in your MongoDB deployment. All connected servers are monitored in a non-blocking manner. Slow to respond or down servers no longer block server discovery, reducing application startup time and time to respond to new or reconfigured servers and replica set failovers. - A unified CRUD API. All official MongoDB drivers now implement a standard CRUD API allowing polyglot developers to move from language to language with ease. - Single source support for Python 2.x and 3.x. PyMongo no longer relies on 2to3 to support Python 3. - A rewritten pure Python BSON implementation, improving performance with pypy and cpython deployments without support for C extensions. - Better support for greenlet based async frameworks including eventlet. - Immutable client, database, and collection classes, avoiding a host of thread safety issues in client applications. PyMongo 3.0 brings a large number of API changes. Be sure to read the changes listed below before upgrading from PyMongo 2.x. .. warning:: PyMongo no longer supports Python 2.4, 2.5, or 3.1. If you must use PyMongo with these versions of Python the 2.x branch of PyMongo will be minimally supported for some time. SONManipulator changes ...................... The :class:`~pymongo.son_manipulator.SONManipulator` API has limitations as a technique for transforming your data. Instead, it is more flexible and straightforward to transform outgoing documents in your own code before passing them to PyMongo, and transform incoming documents after receiving them from PyMongo. Thus the :meth:`~pymongo.database.Database.add_son_manipulator` method is deprecated. PyMongo 3's new CRUD API does **not** apply SON manipulators to documents passed to :meth:`~pymongo.collection.Collection.bulk_write`, :meth:`~pymongo.collection.Collection.insert_one`, :meth:`~pymongo.collection.Collection.insert_many`, :meth:`~pymongo.collection.Collection.update_one`, or :meth:`~pymongo.collection.Collection.update_many`. SON manipulators are **not** applied to documents returned by the new methods :meth:`~pymongo.collection.Collection.find_one_and_delete`, :meth:`~pymongo.collection.Collection.find_one_and_replace`, and :meth:`~pymongo.collection.Collection.find_one_and_update`. SSL/TLS changes ............... When `ssl` is ``True`` the `ssl_cert_reqs` option now defaults to :attr:`ssl.CERT_REQUIRED` if not provided. PyMongo will attempt to load OS provided CA certificates to verify the server, raising :exc:`~pymongo.errors.ConfigurationError` if it cannot. Gevent Support .............. In previous versions, PyMongo supported Gevent in two modes: you could call ``gevent.monkey.patch_socket()`` and pass ``use_greenlets=True`` to :class:`~pymongo.mongo_client.MongoClient`, or you could simply call ``gevent.monkey.patch_all()`` and omit the ``use_greenlets`` argument. In PyMongo 3.0, the ``use_greenlets`` option is gone. To use PyMongo with Gevent simply call ``gevent.monkey.patch_all()``. For more information, see :doc:`PyMongo's Gevent documentation `. :class:`~pymongo.mongo_client.MongoClient` changes .................................................. :class:`~pymongo.mongo_client.MongoClient` is now the one and only client class for a standalone server, mongos, or replica set. It includes the functionality that had been split into ``MongoReplicaSetClient``: it can connect to a replica set, discover all its members, and monitor the set for stepdowns, elections, and reconfigs. :class:`~pymongo.mongo_client.MongoClient` now also supports the full :class:`~pymongo.read_preferences.ReadPreference` API. The obsolete classes ``MasterSlaveConnection``, ``Connection``, and ``ReplicaSetConnection`` are removed. The :class:`~pymongo.mongo_client.MongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :class:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor :class:`~pymongo.errors.ConfigurationError` if the user's credentials are wrong. Instead, the constructor returns immediately and launches the connection process on background threads. The ``connect`` option is added to control whether these threads are started immediately, or when the client is first used. Therefore the ``alive`` method is removed since it no longer provides meaningful information; even if the client is disconnected, it may discover a server in time to fulfill the next operation. In PyMongo 2.x, :class:`~pymongo.mongo_client.MongoClient` accepted a list of standalone MongoDB servers and used the first it could connect to:: MongoClient(['host1.com:27017', 'host2.com:27017']) A list of multiple standalones is no longer supported; if multiple servers are listed they must be members of the same replica set, or mongoses in the same sharded cluster. The behavior for a list of mongoses is changed from "high availability" to "load balancing". Before, the client connected to the lowest-latency mongos in the list, and used it until a network error prompted it to re-evaluate all mongoses' latencies and reconnect to one of them. In PyMongo 3, the client monitors its network latency to all the mongoses continuously, and distributes operations evenly among those with the lowest latency. See :ref:`mongos-load-balancing` for more information. The client methods ``start_request``, ``in_request``, and ``end_request`` are removed, and so is the ``auto_start_request`` option. Requests were designed to make read-your-writes consistency more likely with the ``w=0`` write concern. Additionally, a thread in a request used the same member for all secondary reads in a replica set. To ensure read-your-writes consistency in PyMongo 3.0, do not override the default write concern with ``w=0``, and do not override the default :ref:`read preference ` of PRIMARY. Support for the ``slaveOk`` (or ``slave_okay``), ``safe``, and ``network_timeout`` options has been removed. Use :attr:`~pymongo.read_preferences.ReadPreference.SECONDARY_PREFERRED` instead of slave_okay. Accept the default write concern, acknowledged writes, instead of setting safe=True. Use socketTimeoutMS in place of network_timeout (note that network_timeout was in seconds, where as socketTimeoutMS is milliseconds). The ``max_pool_size`` option has been removed. It is replaced by the ``maxPoolSize`` MongoDB URI option. ``maxPoolSize`` is now a supported URI option in PyMongo and can be passed as a keyword argument. The ``copy_database`` method is removed, see the :doc:`copy_database examples ` for alternatives. The ``disconnect`` method is removed. Use :meth:`~pymongo.mongo_client.MongoClient.close` instead. The ``get_document_class`` method is removed. Use :attr:`~pymongo.mongo_client.MongoClient.codec_options` instead. The ``get_lasterror_options``, ``set_lasterror_options``, and ``unset_lasterror_options`` methods are removed. Write concern options can be passed to :class:`~pymongo.mongo_client.MongoClient` as keyword arguments or MongoDB URI options. The :meth:`~pymongo.mongo_client.MongoClient.get_database` method is added for getting a Database instance with its options configured differently than the MongoClient's. The following read-only attributes have been added: - :attr:`~pymongo.mongo_client.MongoClient.codec_options` The following attributes are now read-only: - :attr:`~pymongo.mongo_client.MongoClient.read_preference` - :attr:`~pymongo.mongo_client.MongoClient.write_concern` The following attributes have been removed: - :attr:`~pymongo.mongo_client.MongoClient.document_class` (use :attr:`~pymongo.mongo_client.MongoClient.codec_options` instead) - :attr:`~pymongo.mongo_client.MongoClient.host` (use :attr:`~pymongo.mongo_client.MongoClient.address` instead) - :attr:`~pymongo.mongo_client.MongoClient.min_wire_version` - :attr:`~pymongo.mongo_client.MongoClient.max_wire_version` - :attr:`~pymongo.mongo_client.MongoClient.port` (use :attr:`~pymongo.mongo_client.MongoClient.address` instead) - :attr:`~pymongo.mongo_client.MongoClient.safe` (use :attr:`~pymongo.mongo_client.MongoClient.write_concern` instead) - :attr:`~pymongo.mongo_client.MongoClient.slave_okay` (use :attr:`~pymongo.mongo_client.MongoClient.read_preference` instead) - :attr:`~pymongo.mongo_client.MongoClient.tag_sets` (use :attr:`~pymongo.mongo_client.MongoClient.read_preference` instead) - :attr:`~pymongo.mongo_client.MongoClient.tz_aware` (use :attr:`~pymongo.mongo_client.MongoClient.codec_options` instead) The following attributes have been renamed: - :attr:`~pymongo.mongo_client.MongoClient.secondary_acceptable_latency_ms` is now :attr:`~pymongo.mongo_client.MongoClient.local_threshold_ms` and is now read-only. :class:`~pymongo.cursor.Cursor` changes ....................................... The ``conn_id`` property is renamed to :attr:`~pymongo.cursor.Cursor.address`. Cursor management changes ......................... :class:`~pymongo.cursor_manager.CursorManager` and :meth:`~pymongo.mongo_client.MongoClient.set_cursor_manager` are no longer deprecated. If you subclass :class:`~pymongo.cursor_manager.CursorManager` your implementation of :meth:`~pymongo.cursor_manager.CursorManager.close` must now take a second parameter, `address`. The ``BatchCursorManager`` class is removed. The second parameter to :meth:`~pymongo.mongo_client.MongoClient.close_cursor` is renamed from ``_conn_id`` to ``address``. :meth:`~pymongo.mongo_client.MongoClient.kill_cursors` now accepts an `address` parameter. :class:`~pymongo.database.Database` changes ........................................... The ``connection`` property is renamed to :attr:`~pymongo.database.Database.client`. The following read-only attributes have been added: - :attr:`~pymongo.database.Database.codec_options` The following attributes are now read-only: - :attr:`~pymongo.database.Database.read_preference` - :attr:`~pymongo.database.Database.write_concern` Use :meth:`~pymongo.mongo_client.MongoClient.get_database` for getting a Database instance with its options configured differently than the MongoClient's. The following attributes have been removed: - :attr:`~pymongo.database.Database.safe` - :attr:`~pymongo.database.Database.secondary_acceptable_latency_ms` - :attr:`~pymongo.database.Database.slave_okay` - :attr:`~pymongo.database.Database.tag_sets` The following methods have been added: - :meth:`~pymongo.database.Database.get_collection` The following methods have been changed: - :meth:`~pymongo.database.Database.command`. Support for `as_class`, `uuid_subtype`, `tag_sets`, and `secondary_acceptable_latency_ms` have been removed. You can instead pass an instance of :class:`~bson.codec_options.CodecOptions` as `codec_options` and an instance of a read preference class from :mod:`~pymongo.read_preferences` as `read_preference`. The `fields` and `compile_re` options are also removed. The `fields` options was undocumented and never really worked. Regular expressions are always decoded to :class:`~bson.regex.Regex`. The following methods have been deprecated: - :meth:`~pymongo.database.Database.add_son_manipulator` The following methods have been removed: The ``get_lasterror_options``, ``set_lasterror_options``, and ``unset_lasterror_options`` methods have been removed. Use :class:`~pymongo.write_concern.WriteConcern` with :meth:`~pymongo.mongo_client.MongoClient.get_database` instead. :class:`~pymongo.collection.Collection` changes ............................................... The following read-only attributes have been added: - :attr:`~pymongo.collection.Collection.codec_options` The following attributes are now read-only: - :attr:`~pymongo.collection.Collection.read_preference` - :attr:`~pymongo.collection.Collection.write_concern` Use :meth:`~pymongo.database.Database.get_collection` or :meth:`~pymongo.collection.Collection.with_options` for getting a Collection instance with its options configured differently than the Database's. The following attributes have been removed: - :attr:`~pymongo.collection.Collection.safe` - :attr:`~pymongo.collection.Collection.secondary_acceptable_latency_ms` - :attr:`~pymongo.collection.Collection.slave_okay` - :attr:`~pymongo.collection.Collection.tag_sets` The following methods have been added: - :meth:`~pymongo.collection.Collection.bulk_write` - :meth:`~pymongo.collection.Collection.insert_one` - :meth:`~pymongo.collection.Collection.insert_many` - :meth:`~pymongo.collection.Collection.update_one` - :meth:`~pymongo.collection.Collection.update_many` - :meth:`~pymongo.collection.Collection.replace_one` - :meth:`~pymongo.collection.Collection.delete_one` - :meth:`~pymongo.collection.Collection.delete_many` - :meth:`~pymongo.collection.Collection.find_one_and_delete` - :meth:`~pymongo.collection.Collection.find_one_and_replace` - :meth:`~pymongo.collection.Collection.find_one_and_update` - :meth:`~pymongo.collection.Collection.with_options` - :meth:`~pymongo.collection.Collection.create_indexes` - :meth:`~pymongo.collection.Collection.list_indexes` The following methods have changed: - :meth:`~pymongo.collection.Collection.aggregate` now **always** returns an instance of :class:`~pymongo.command_cursor.CommandCursor`. See the documentation for all options. - :meth:`~pymongo.collection.Collection.count` now optionally takes a filter argument, as well as other options supported by the count command. - :meth:`~pymongo.collection.Collection.distinct` now optionally takes a filter argument. - :meth:`~pymongo.collection.Collection.create_index` no longer caches indexes, therefore the `cache_for` parameter has been removed. It also no longer supports the `bucket_size` and `drop_dups` aliases for `bucketSize` and `dropDups`. The following methods are deprecated: - :meth:`~pymongo.collection.Collection.save` - :meth:`~pymongo.collection.Collection.insert` - :meth:`~pymongo.collection.Collection.update` - :meth:`~pymongo.collection.Collection.remove` - :meth:`~pymongo.collection.Collection.find_and_modify` - :meth:`~pymongo.collection.Collection.ensure_index` The following methods have been removed: The ``get_lasterror_options``, ``set_lasterror_options``, and ``unset_lasterror_options`` methods have been removed. Use :class:`~pymongo.write_concern.WriteConcern` with :meth:`~pymongo.collection.Collection.with_options` instead. Changes to :meth:`~pymongo.collection.Collection.find` and :meth:`~pymongo.collection.Collection.find_one` `````````````````````````````````````````````````````````````````````````````````````````````````````````` The following find/find_one options have been renamed: These renames only affect your code if you passed these as keyword arguments, like find(fields=['fieldname']). If you passed only positional parameters these changes are not significant for your application. - spec -> filter - fields -> projection - partial -> allow_partial_results The following find/find_one options have been added: - cursor_type (see :class:`~pymongo.cursor.CursorType` for values) - oplog_replay - modifiers The following find/find_one options have been removed: - network_timeout (use :meth:`~pymongo.cursor.Cursor.max_time_ms` instead) - slave_okay (use one of the read preference classes from :mod:`~pymongo.read_preferences` and :meth:`~pymongo.collection.Collection.with_options` instead) - read_preference (use :meth:`~pymongo.collection.Collection.with_options` instead) - tag_sets (use one of the read preference classes from :mod:`~pymongo.read_preferences` and :meth:`~pymongo.collection.Collection.with_options` instead) - secondary_acceptable_latency_ms (use the `localThresholdMS` URI option instead) - max_scan (use the new `modifiers` option instead) - snapshot (use the new `modifiers` option instead) - tailable (use the new `cursor_type` option instead) - await_data (use the new `cursor_type` option instead) - exhaust (use the new `cursor_type` option instead) - as_class (use :meth:`~pymongo.collection.Collection.with_options` with :class:`~bson.codec_options.CodecOptions` instead) - compile_re (BSON regular expressions are always decoded to :class:`~bson.regex.Regex`) The following find/find_one options are deprecated: - manipulate The following renames need special handling. - timeout -> no_cursor_timeout - The default for `timeout` was True. The default for `no_cursor_timeout` is False. If you were previously passing False for `timeout` you must pass **True** for `no_cursor_timeout` to keep the previous behavior. :mod:`~pymongo.errors` changes .............................. The exception classes ``UnsupportedOption`` and ``TimeoutError`` are deleted. :mod:`~gridfs` changes ...................... Since PyMongo 1.6, methods ``open`` and ``close`` of :class:`~gridfs.GridFS` raised an ``UnsupportedAPI`` exception, as did the entire ``GridFile`` class. The unsupported methods, the class, and the exception are all deleted. :mod:`~bson` changes .................... The `compile_re` option is removed from all methods that accepted it in :mod:`~bson` and :mod:`~bson.json_util`. Additionally, it is removed from :meth:`~pymongo.collection.Collection.find`, :meth:`~pymongo.collection.Collection.find_one`, :meth:`~pymongo.collection.Collection.aggregate`, :meth:`~pymongo.database.Database.command`, and so on. PyMongo now always represents BSON regular expressions as :class:`~bson.regex.Regex` objects. This prevents errors for incompatible patterns, see `PYTHON-500`_. Use :meth:`~bson.regex.Regex.try_compile` to attempt to convert from a BSON regular expression to a Python regular expression object. PyMongo now decodes the int64 BSON type to :class:`~bson.int64.Int64`, a trivial wrapper around long (in python 2.x) or int (in python 3.x). This allows BSON int64 to be round tripped without losing type information in python 3. Note that if you store a python long (or a python int larger than 4 bytes) it will be returned from PyMongo as :class:`~bson.int64.Int64`. The `as_class`, `tz_aware`, and `uuid_subtype` options are removed from all BSON encoding and decoding methods. Use :class:`~bson.codec_options.CodecOptions` to configure these options. The APIs affected are: - :func:`~bson.decode_all` - :func:`~bson.decode_iter` - :func:`~bson.decode_file_iter` - :meth:`~bson.BSON.encode` - :meth:`~bson.BSON.decode` This is a breaking change for any application that uses the BSON API directly and changes any of the named parameter defaults. No changes are required for applications that use the default values for these options. The behavior remains the same. .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500 Issues Resolved ............... See the `PyMongo 3.0 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.0 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12501 Changes in Version 2.9.5 ------------------------ Version 2.9.5 works around ssl module deprecations in Python 3.6, and expected future ssl module deprecations. It also fixes bugs found since the release of 2.9.4. - Use ssl.SSLContext and ssl.PROTOCOL_TLS_CLIENT when available. - Fixed a C extensions build issue when the interpreter was built with -std=c99 - Fixed various build issues with MinGW32. - Fixed a write concern bug in :meth:`~pymongo.database.Database.add_user` and :meth:`~pymongo.database.Database.remove_user` when connected to MongoDB 3.2+ - Fixed various test failures related to changes in gevent, MongoDB, and our CI test environment. Issues Resolved ............... See the `PyMongo 2.9.5 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.9.5 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=17605 Changes in Version 2.9.4 ------------------------ Version 2.9.4 fixes issues reported since the release of 2.9.3. - Fixed __repr__ for closed instances of :class:`~pymongo.mongo_client.MongoClient`. - Fixed :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` handling of uuidRepresentation. - Fixed building and testing the documentation with python 3.x. - New documentation for :doc:`examples/tls` and :doc:`atlas`. Issues Resolved ............... See the `PyMongo 2.9.4 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.9.4 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16885 Changes in Version 2.9.3 ------------------------ Version 2.9.3 fixes a few issues reported since the release of 2.9.2 including thread safety issues in :meth:`~pymongo.collection.Collection.ensure_index`, :meth:`~pymongo.collection.Collection.drop_index`, and :meth:`~pymongo.collection.Collection.drop_indexes`. Issues Resolved ............... See the `PyMongo 2.9.3 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.9.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16539 Changes in Version 2.9.2 ------------------------ Version 2.9.2 restores Python 3.1 support, which was broken in PyMongo 2.8. It improves an error message when decoding BSON as well as fixes a couple other issues including :meth:`~pymongo.collection.Collection.aggregate` ignoring :attr:`~pymongo.collection.Collection.codec_options` and :meth:`~pymongo.database.Database.command` raising a superfluous `DeprecationWarning`. Issues Resolved ............... See the `PyMongo 2.9.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.9.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16303 Changes in Version 2.9.1 ------------------------ Version 2.9.1 fixes two interrupt handling issues in the C extensions and adapts a test case for a behavior change in MongoDB 3.2. Issues Resolved ............... See the `PyMongo 2.9.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.9.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=16208 Changes in Version 2.9 ---------------------- Version 2.9 provides an upgrade path to PyMongo 3.x. Most of the API changes from PyMongo 3.0 have been backported in a backward compatible way, allowing applications to be written against PyMongo >= 2.9, rather then PyMongo 2.x or PyMongo 3.x. See the :doc:`/migrate-to-pymongo3` for detailed examples. .. note:: There are a number of new deprecations in this release for features that were removed in PyMongo 3.0. :class:`~pymongo.mongo_client.MongoClient`: - :attr:`~pymongo.mongo_client.MongoClient.host` - :attr:`~pymongo.mongo_client.MongoClient.port` - :attr:`~pymongo.mongo_client.MongoClient.use_greenlets` - :attr:`~pymongo.mongo_client.MongoClient.document_class` - :attr:`~pymongo.mongo_client.MongoClient.tz_aware` - :attr:`~pymongo.mongo_client.MongoClient.secondary_acceptable_latency_ms` - :attr:`~pymongo.mongo_client.MongoClient.tag_sets` - :attr:`~pymongo.mongo_client.MongoClient.uuid_subtype` - :meth:`~pymongo.mongo_client.MongoClient.disconnect` - :meth:`~pymongo.mongo_client.MongoClient.alive` :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`: - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.use_greenlets` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.document_class` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.tz_aware` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.secondary_acceptable_latency_ms` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.tag_sets` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.uuid_subtype` - :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.alive` :class:`~pymongo.database.Database`: - :attr:`~pymongo.database.Database.secondary_acceptable_latency_ms` - :attr:`~pymongo.database.Database.tag_sets` - :attr:`~pymongo.database.Database.uuid_subtype` :class:`~pymongo.collection.Collection`: - :attr:`~pymongo.collection.Collection.secondary_acceptable_latency_ms` - :attr:`~pymongo.collection.Collection.tag_sets` - :attr:`~pymongo.collection.Collection.uuid_subtype` .. warning:: In previous versions of PyMongo, changing the value of :attr:`~pymongo.mongo_client.MongoClient.document_class` changed the behavior of all existing instances of :class:`~pymongo.collection.Collection`:: >>> coll = client.test.test >>> coll.find_one() {u'_id': ObjectId('5579dc7cfba5220cc14d9a18')} >>> from bson.son import SON >>> client.document_class = SON >>> coll.find_one() SON([(u'_id', ObjectId('5579dc7cfba5220cc14d9a18'))]) The document_class setting is now configurable at the client, database, collection, and per-operation level. This required breaking the existing behavior. To change the document class per operation in a forward compatible way use :meth:`~pymongo.collection.Collection.with_options`:: >>> coll.find_one() {u'_id': ObjectId('5579dc7cfba5220cc14d9a18')} >>> from bson.codec_options import CodecOptions >>> coll.with_options(CodecOptions(SON)).find_one() SON([(u'_id', ObjectId('5579dc7cfba5220cc14d9a18'))]) Issues Resolved ............... See the `PyMongo 2.9 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.9 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=14795 Changes in Version 2.8.1 ------------------------ Version 2.8.1 fixes a number of issues reported since the release of PyMongo 2.8. It is a recommended upgrade for all users of PyMongo 2.x. Issues Resolved ............... See the `PyMongo 2.8.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.8.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=15324 Changes in Version 2.8 ---------------------- Version 2.8 is a major release that provides full support for MongoDB 3.0 and fixes a number of bugs. Special thanks to Don Mitchell, Ximing, Can Zhang, Sergey Azovskov, and Heewa Barfchin for their contributions to this release. Highlights include: - Support for the SCRAM-SHA-1 authentication mechanism (new in MongoDB 3.0). - JSON decoder support for the new $numberLong and $undefined types. - JSON decoder support for the $date type as an ISO-8601 string. - Support passing an index name to :meth:`~pymongo.cursor.Cursor.hint`. - The :meth:`~pymongo.cursor.Cursor.count` method will use a hint if one has been provided through :meth:`~pymongo.cursor.Cursor.hint`. - A new socketKeepAlive option for the connection pool. - New generator based BSON decode functions, :func:`~bson.decode_iter` and :func:`~bson.decode_file_iter`. - Internal changes to support alternative storage engines like wiredtiger. .. note:: There are a number of deprecations in this release for features that will be removed in PyMongo 3.0. These include: - :meth:`~pymongo.mongo_client.MongoClient.start_request` - :meth:`~pymongo.mongo_client.MongoClient.in_request` - :meth:`~pymongo.mongo_client.MongoClient.end_request` - :meth:`~pymongo.mongo_client.MongoClient.copy_database` - :meth:`~pymongo.database.Database.error` - :meth:`~pymongo.database.Database.last_status` - :meth:`~pymongo.database.Database.previous_error` - :meth:`~pymongo.database.Database.reset_error_history` - :class:`~pymongo.master_slave_connection.MasterSlaveConnection` The JSON format for :class:`~bson.timestamp.Timestamp` has changed from '{"t": , "i": }' to '{"$timestamp": {"t": , "i": }}'. This new format will be decoded to an instance of :class:`~bson.timestamp.Timestamp`. The old format will continue to be decoded to a python dict as before. Encoding to the old format is no longer supported as it was never correct and loses type information. Issues Resolved ............... See the `PyMongo 2.8 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.8 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=14223 Changes in Version 2.7.2 ------------------------ Version 2.7.2 includes fixes for upsert reporting in the bulk API for MongoDB versions previous to 2.6, a regression in how son manipulators are applied in :meth:`~pymongo.collection.Collection.insert`, a few obscure connection pool semaphore leaks, and a few other minor issues. See the list of issues resolved for full details. Issues Resolved ............... See the `PyMongo 2.7.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.7.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=14005 Changes in Version 2.7.1 ------------------------ Version 2.7.1 fixes a number of issues reported since the release of 2.7, most importantly a fix for creating indexes and manipulating users through mongos versions older than 2.4.0. Issues Resolved ............... See the `PyMongo 2.7.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.7.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=13823 Changes in Version 2.7 ---------------------- PyMongo 2.7 is a major release with a large number of new features and bug fixes. Highlights include: - Full support for MongoDB 2.6. - A new :doc:`bulk write operations API `. - Support for server side query timeouts using :meth:`~pymongo.cursor.Cursor.max_time_ms`. - Support for writing :meth:`~pymongo.collection.Collection.aggregate` output to a collection. - A new :meth:`~pymongo.collection.Collection.parallel_scan` helper. - :class:`~pymongo.errors.OperationFailure` and its subclasses now include a :attr:`~pymongo.errors.OperationFailure.details` attribute with complete error details from the server. - A new GridFS :meth:`~gridfs.GridFS.find` method that returns a :class:`~gridfs.grid_file.GridOutCursor`. - Greatly improved :doc:`support for mod_wsgi ` when using PyMongo's C extensions. Read `Jesse's blog post `_ for details. - Improved C extension support for ARM little endian. Breaking changes ................ Version 2.7 drops support for replica sets running MongoDB versions older than 1.6.2. Issues Resolved ............... See the `PyMongo 2.7 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.7 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12892 Changes in Version 2.6.3 ------------------------ Version 2.6.3 fixes issues reported since the release of 2.6.2, most importantly a semaphore leak when a connection to the server fails. Issues Resolved ............... See the `PyMongo 2.6.3 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.6.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=13098 Changes in Version 2.6.2 ------------------------ Version 2.6.2 fixes a :exc:`TypeError` problem when max_pool_size=None is used in Python 3. Issues Resolved ............... See the `PyMongo 2.6.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.6.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12910 Changes in Version 2.6.1 ------------------------ Version 2.6.1 fixes a reference leak in the :meth:`~pymongo.collection.Collection.insert` method. Issues Resolved ............... See the `PyMongo 2.6.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.6.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12905 Changes in Version 2.6 ---------------------- Version 2.6 includes some frequently requested improvements and adds support for some early MongoDB 2.6 features. Special thanks go to Justin Patrin for his work on the connection pool in this release. Important new features: - The ``max_pool_size`` option for :class:`~pymongo.mongo_client.MongoClient` and :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` now actually caps the number of sockets the pool will open concurrently. Once the pool has reached :attr:`~pymongo.mongo_client.MongoClient.max_pool_size` operations will block waiting for a socket to become available. If ``waitQueueTimeoutMS`` is set, an operation that blocks waiting for a socket will raise :exc:`~pymongo.errors.ConnectionFailure` after the timeout. By default ``waitQueueTimeoutMS`` is not set. See :ref:`connection-pooling` for more information. - The :meth:`~pymongo.collection.Collection.insert` method automatically splits large batches of documents into multiple insert messages based on :attr:`~pymongo.mongo_client.MongoClient.max_message_size` - Support for the exhaust cursor flag. See :meth:`~pymongo.collection.Collection.find` for details and caveats. - Support for the PLAIN and MONGODB-X509 authentication mechanisms. See :doc:`the authentication docs ` for more information. - Support aggregation output as a :class:`~pymongo.cursor.Cursor`. See :meth:`~pymongo.collection.Collection.aggregate` for details. .. warning:: SIGNIFICANT BEHAVIOR CHANGE in 2.6. Previously, `max_pool_size` would limit only the idle sockets the pool would hold onto, not the number of open sockets. The default has also changed, from 10 to 100. If you pass a value for ``max_pool_size`` make sure it is large enough for the expected load. (Sockets are only opened when needed, so there is no cost to having a ``max_pool_size`` larger than necessary. Err towards a larger value.) If your application accepts the default, continue to do so. See :ref:`connection-pooling` for more information. Issues Resolved ............... See the `PyMongo 2.6 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.6 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12380 Changes in Version 2.5.2 ------------------------ Version 2.5.2 fixes a NULL pointer dereference issue when decoding an invalid :class:`~bson.dbref.DBRef`. Issues Resolved ............... See the `PyMongo 2.5.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.5.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12581 Changes in Version 2.5.1 ------------------------ Version 2.5.1 is a minor release that fixes issues discovered after the release of 2.5. Most importantly, this release addresses some race conditions in replica set monitoring. Issues Resolved ............... See the `PyMongo 2.5.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.5.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12484 Changes in Version 2.5 ---------------------- Version 2.5 includes changes to support new features in MongoDB 2.4. Important new features: - Support for :ref:`GSSAPI (Kerberos) authentication `. - Support for SSL certificate validation with hostname matching. - Support for delegated and role based authentication. - New GEOSPHERE (2dsphere) and HASHED index constants. .. note:: :meth:`~pymongo.database.Database.authenticate` now raises a subclass of :class:`~pymongo.errors.PyMongoError` if authentication fails due to invalid credentials or configuration issues. Issues Resolved ............... See the `PyMongo 2.5 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.5 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=11981 Changes in Version 2.4.2 ------------------------ Version 2.4.2 is a minor release that fixes issues discovered after the release of 2.4.1. Most importantly, PyMongo will no longer select a replica set member for read operations that is not in primary or secondary state. Issues Resolved ............... See the `PyMongo 2.4.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.4.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12299 Changes in Version 2.4.1 ------------------------ Version 2.4.1 is a minor release that fixes issues discovered after the release of 2.4. Most importantly, this release fixes a regression using :meth:`~pymongo.collection.Collection.aggregate`, and possibly other commands, with mongos. Issues Resolved ............... See the `PyMongo 2.4.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.4.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=12286 Changes in Version 2.4 ---------------------- Version 2.4 includes a few important new features and a large number of bug fixes. Important new features: - New :class:`~pymongo.mongo_client.MongoClient` and :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` classes - these connection classes do acknowledged write operations (previously referred to as 'safe' writes) by default. :class:`~pymongo.connection.Connection` and :class:`~pymongo.replica_set_connection.ReplicaSetConnection` are deprecated but still support the old default fire-and-forget behavior. - A new write concern API implemented as a :attr:`~pymongo.collection.Collection.write_concern` attribute on the connection, :class:`~pymongo.database.Database`, or :class:`~pymongo.collection.Collection` classes. - :class:`~pymongo.mongo_client.MongoClient` (and :class:`~pymongo.connection.Connection`) now support Unix Domain Sockets. - :class:`~pymongo.cursor.Cursor` can be copied with functions from the :mod:`copy` module. - The :meth:`~pymongo.database.Database.set_profiling_level` method now supports a `slow_ms` option. - The replica set monitor task (used by :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` and :class:`~pymongo.replica_set_connection.ReplicaSetConnection`) is a daemon thread once again, meaning you won't have to call :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.close` before exiting the python interactive shell. .. warning:: The constructors for :class:`~pymongo.mongo_client.MongoClient`, :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`, :class:`~pymongo.connection.Connection`, and :class:`~pymongo.replica_set_connection.ReplicaSetConnection` now raise :exc:`~pymongo.errors.ConnectionFailure` instead of its subclass :exc:`~pymongo.errors.AutoReconnect` if the server is unavailable. Applications that expect to catch :exc:`~pymongo.errors.AutoReconnect` should now catch :exc:`~pymongo.errors.ConnectionFailure` while creating a new connection. Issues Resolved ............... See the `PyMongo 2.4 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.4 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=11485 Changes in Version 2.3 ---------------------- Version 2.3 adds support for new features and behavior changes in MongoDB 2.2. Important New Features: - Support for expanded read preferences including directing reads to tagged servers - See :ref:`secondary-reads` for more information. - Support for mongos failover. - A new :meth:`~pymongo.collection.Collection.aggregate` method to support MongoDB's new `aggregation framework `_. - Support for legacy Java and C# byte order when encoding and decoding UUIDs. - Support for connecting directly to an arbiter. .. warning:: Starting with MongoDB 2.2 the getLastError command requires authentication when the server's `authentication features `_ are enabled. Changes to PyMongo were required to support this behavior change. Users of authentication must upgrade to PyMongo 2.3 (or newer) for "safe" write operations to function correctly. Issues Resolved ............... See the `PyMongo 2.3 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.3 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=11146 Changes in Version 2.2.1 ------------------------ Version 2.2.1 is a minor release that fixes issues discovered after the release of 2.2. Most importantly, this release fixes an incompatibility with mod_wsgi 2.x that could cause connections to leak. Users of mod_wsgi 2.x are strongly encouraged to upgrade from PyMongo 2.2. Issues Resolved ............... See the `PyMongo 2.2.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.2.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=11185 Changes in Version 2.2 ---------------------- Version 2.2 adds a few more frequently requested features and fixes a number of bugs. Special thanks go to Alex Grönholm for his contributions to Python 3 support and maintaining the original pymongo3 port. Christoph Simon, Wouter Bolsterlee, Mike O'Brien, and Chris Tompkinson also contributed to this release. Important New Features: - Support for Python 3 - See the :doc:`python3` for more information. - Support for Gevent - See :doc:`examples/gevent` for more information. - Improved connection pooling. See `PYTHON-287 `_. .. warning:: A number of methods and method parameters that were deprecated in PyMongo 1.9 or older versions have been removed in this release. The full list of changes can be found in the following JIRA ticket: https://jira.mongodb.org/browse/PYTHON-305 BSON module aliases from the pymongo package that were deprecated in PyMongo 1.9 have also been removed in this release. See the following JIRA ticket for details: https://jira.mongodb.org/browse/PYTHON-304 As a result of this cleanup some minor code changes may be required to use this release. Issues Resolved ............... See the `PyMongo 2.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=10584 Changes in Version 2.1.1 ------------------------ Version 2.1.1 is a minor release that fixes a few issues discovered after the release of 2.1. You can now use :class:`~pymongo.replica_set_connection.ReplicaSetConnection` to run inline map reduce commands on secondaries. See :meth:`~pymongo.collection.Collection.inline_map_reduce` for details. Special thanks go to Samuel Clay and Ross Lawley for their contributions to this release. Issues Resolved ............... See the `PyMongo 2.1.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.1.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?version=11081&styleName=Html&projectId=10004 Changes in Version 2.1 ---------------------- Version 2.1 adds a few frequently requested features and includes the usual round of bug fixes and improvements. Special thanks go to Alexey Borzenkov, Dan Crosta, Kostya Rybnikov, Flavio Percoco Premoli, Jonas Haag, and Jesse Davis for their contributions to this release. Important New Features: - ReplicaSetConnection - :class:`~pymongo.replica_set_connection.ReplicaSetConnection` can be used to distribute reads to secondaries in a replica set. It supports automatic failover handling and periodically checks the state of the replica set to handle issues like primary stepdown or secondaries being removed for backup operations. Read preferences are defined through :class:`~pymongo.read_preferences.ReadPreference`. - PyMongo supports the new BSON binary subtype 4 for UUIDs. The default subtype to use can be set through :attr:`~pymongo.collection.Collection.uuid_subtype` The current default remains :attr:`~bson.binary.OLD_UUID_SUBTYPE` but will be changed to :attr:`~bson.binary.UUID_SUBTYPE` in a future release. - The getLastError option 'w' can be set to a string, allowing for options like "majority" available in newer version of MongoDB. - Added support for the MongoDB URI options socketTimeoutMS and connectTimeoutMS. - Added support for the ContinueOnError insert flag. - Added basic SSL support. - Added basic support for Jython. - Secondaries can be used for :meth:`~pymongo.cursor.Cursor.count`, :meth:`~pymongo.cursor.Cursor.distinct`, :meth:`~pymongo.collection.Collection.group`, and querying :class:`~gridfs.GridFS`. - Added document_class and tz_aware options to :class:`~pymongo.master_slave_connection.MasterSlaveConnection` Issues Resolved ............... See the `PyMongo 2.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=10583 Changes in Version 2.0.1 ------------------------ Version 2.0.1 fixes a regression in :class:`~gridfs.grid_file.GridIn` when writing pre-chunked strings. Thanks go to Alexey Borzenkov for reporting the issue and submitting a patch. Issues Resolved ............... - `PYTHON-271 `_: Regression in GridFS leads to serious loss of data. Changes in Version 2.0 ---------------------- Version 2.0 adds a large number of features and fixes a number of issues. Special thanks go to James Murty, Abhay Vardhan, David Pisoni, Ryan Smith-Roberts, Andrew Pendleton, Mher Movsisyan, Reed O'Brien, Michael Schurter, Josip Delic and Jonas Haag for their contributions to this release. Important New Features: - PyMongo now performs automatic per-socket database authentication. You no longer have to re-authenticate for each new thread or after a replica set failover. Authentication credentials are cached by the driver until the application calls :meth:`~pymongo.database.Database.logout`. - slave_okay can be set independently at the connection, database, collection or query level. Each level will inherit the slave_okay setting from the previous level and each level can override the previous level's setting. - safe and getLastError options (e.g. w, wtimeout, etc.) can be set independently at the connection, database, collection or query level. Each level will inherit settings from the previous level and each level can override the previous level's setting. - PyMongo now supports the `await_data` and `partial` cursor flags. If the `await_data` flag is set on a `tailable` cursor the server will block for some extra time waiting for more data to return. The `partial` flag tells a mongos to return partial data for a query if not all shards are available. - :meth:`~pymongo.collection.Collection.map_reduce` will accept a `dict` or instance of :class:`~bson.son.SON` as the `out` parameter. - The URI parser has been moved into its own module and can be used directly by application code. - AutoReconnect exception now provides information about the error that actually occured instead of a generic failure message. - A number of new helper methods have been added with options for setting and unsetting cursor flags, re-indexing a collection, fsync and locking a server, and getting the server's current operations. API changes: - If only one host:port pair is specified :class:`~pymongo.connection.Connection` will make a direct connection to only that host. Please note that `slave_okay` must be `True` in order to query from a secondary. - If more than one host:port pair is specified or the `replicaset` option is used PyMongo will treat the specified host:port pair(s) as a seed list and connect using replica set behavior. .. warning:: The default subtype for :class:`~bson.binary.Binary` has changed from :const:`~bson.binary.OLD_BINARY_SUBTYPE` (2) to :const:`~bson.binary.BINARY_SUBTYPE` (0). Issues Resolved ............... See the `PyMongo 2.0 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.0 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=10274 Changes in Version 1.11 ----------------------- Version 1.11 adds a few new features and fixes a few more bugs. New Features: - Basic IPv6 support: pymongo prefers IPv4 but will try IPv6. You can also specify an IPv6 address literal in the `host` parameter or a MongoDB URI provided it is enclosed in '[' and ']'. - max_pool_size option: previously pymongo had a hard coded pool size of 10 connections. With this change you can specify a different pool size as a parameter to :class:`~pymongo.connection.Connection` (max_pool_size=) or in the MongoDB URI (maxPoolSize=). - Find by metadata in GridFS: You can know specify query fields as keyword parameters for :meth:`~gridfs.GridFS.get_version` and :meth:`~gridfs.GridFS.get_last_version`. - Per-query slave_okay option: slave_okay=True is now a valid keyword argument for :meth:`~pymongo.collection.Collection.find` and :meth:`~pymongo.collection.Collection.find_one`. API changes: - :meth:`~pymongo.database.Database.validate_collection` now returns a dict instead of a string. This change was required to deal with an API change on the server. This method also now takes the optional `scandata` and `full` parameters. See the documentation for more details. .. warning:: The `pool_size`, `auto_start_request`, and `timeout` parameters for :class:`~pymongo.connection.Connection` have been completely removed in this release. They were deprecated in pymongo-1.4 and have had no effect since then. Please make sure that your code doesn't currently pass these parameters when creating a Connection instance. Issues resolved ............... - `PYTHON-241 `_: Support setting slaveok at the cursor level. - `PYTHON-240 `_: Queries can sometimes permanently fail after a replica set fail over. - `PYTHON-238 `_: error after few million requests - `PYTHON-237 `_: Basic IPv6 support. - `PYTHON-236 `_: Restore option to specify pool size in Connection. - `PYTHON-212 `_: pymongo does not recover after stale config - `PYTHON-138 `_: Find method for GridFS Changes in Version 1.10.1 ------------------------- Version 1.10.1 is primarily a bugfix release. It fixes a regression in version 1.10 that broke pickling of ObjectIds. A number of other bugs have been fixed as well. There are two behavior changes to be aware of: - If a read slave raises :class:`~pymongo.errors.AutoReconnect` :class:`~pymongo.master_slave_connection.MasterSlaveConnection` will now retry the query on each slave until it is successful or all slaves have raised :class:`~pymongo.errors.AutoReconnect`. Any other exception will immediately be raised. The order that the slaves are tried is random. Previously the read would be sent to one randomly chosen slave and :class:`~pymongo.errors.AutoReconnect` was immediately raised in case of a connection failure. - A Python `long` is now always BSON encoded as an int64. Previously the encoding was based only on the value of the field and a `long` with a value less than `2147483648` or greater than `-2147483649` would always be BSON encoded as an int32. Issues resolved ............... - `PYTHON-234 `_: Fix setup.py to raise exception if any when building extensions - `PYTHON-233 `_: Add information to build and test with extensions on windows - `PYTHON-232 `_: Traceback when hashing a DBRef instance - `PYTHON-231 `_: Traceback when pickling a DBRef instance - `PYTHON-230 `_: Pickled ObjectIds are not compatible between pymongo 1.9 and 1.10 - `PYTHON-228 `_: Cannot pickle bson.ObjectId - `PYTHON-227 `_: Traceback when calling find() on system.js - `PYTHON-216 `_: MasterSlaveConnection is missing disconnect() method - `PYTHON-186 `_: When storing integers, type is selected according to value instead of type - `PYTHON-173 `_: as_class option is not propogated by Cursor.clone - `PYTHON-113 `_: Redunducy in MasterSlaveConnection Changes in Version 1.10 ----------------------- Version 1.10 includes changes to support new features in MongoDB 1.8.x. Highlights include a modified map/reduce API including an inline map/reduce helper method, a new find_and_modify helper, and the ability to query the server for the maximum BSON document size it supports. - added :meth:`~pymongo.collection.Collection.find_and_modify`. - added :meth:`~pymongo.collection.Collection.inline_map_reduce`. - changed :meth:`~pymongo.collection.Collection.map_reduce`. .. warning:: MongoDB versions greater than 1.7.4 no longer generate temporary collections for map/reduce results. An output collection name must be provided and the output will replace any existing output collection with the same name. :meth:`~pymongo.collection.Collection.map_reduce` now requires the `out` parameter. Issues resolved ............... - PYTHON-225: :class:`~pymongo.objectid.ObjectId` class definition should use __slots__. - PYTHON-223: Documentation fix. - PYTHON-220: Documentation fix. - PYTHON-219: KeyError in :meth:`~pymongo.collection.Collection.find_and_modify` - PYTHON-213: Query server for maximum BSON document size. - PYTHON-208: Fix :class:`~pymongo.connection.Connection` __repr__. - PYTHON-207: Changes to Map/Reduce API. - PYTHON-205: Accept slaveOk in the URI to match the URI docs. - PYTHON-203: When slave_okay=True and we only specify one host don't autodetect other set members. - PYTHON-194: Show size when whining about a document being too large. - PYTHON-184: Raise :class:`~pymongo.errors.DuplicateKeyError` for duplicate keys in capped collections. - PYTHON-178: Don't segfault when trying to encode a recursive data structure. - PYTHON-177: Don't segfault when decoding dicts with broken iterators. - PYTHON-172: Fix a typo. - PYTHON-170: Add :meth:`~pymongo.collection.Collection.find_and_modify`. - PYTHON-169: Support deepcopy of DBRef. - PYTHON-167: Duplicate of PYTHON-166. - PYTHON-166: Fixes a concurrency issue. - PYTHON-158: Add code and err string to `db assertion` messages. Changes in Version 1.9 ---------------------- Version 1.9 adds a new package to the PyMongo distribution, :mod:`bson`. :mod:`bson` contains all of the `BSON `_ encoding and decoding logic, and the BSON types that were formerly in the :mod:`pymongo` package. The following modules have been renamed: - :mod:`pymongo.bson` -> :mod:`bson` - :mod:`pymongo._cbson` -> :mod:`bson._cbson` and :mod:`pymongo._cmessage` - :mod:`pymongo.binary` -> :mod:`bson.binary` - :mod:`pymongo.code` -> :mod:`bson.code` - :mod:`pymongo.dbref` -> :mod:`bson.dbref` - :mod:`pymongo.json_util` -> :mod:`bson.json_util` - :mod:`pymongo.max_key` -> :mod:`bson.max_key` - :mod:`pymongo.min_key` -> :mod:`bson.min_key` - :mod:`pymongo.objectid` -> :mod:`bson.objectid` - :mod:`pymongo.son` -> :mod:`bson.son` - :mod:`pymongo.timestamp` -> :mod:`bson.timestamp` - :mod:`pymongo.tz_util` -> :mod:`bson.tz_util` In addition, the following exception classes have been renamed: - :class:`pymongo.errors.InvalidBSON` -> :class:`bson.errors.InvalidBSON` - :class:`pymongo.errors.InvalidStringData` -> :class:`bson.errors.InvalidStringData` - :class:`pymongo.errors.InvalidDocument` -> :class:`bson.errors.InvalidDocument` - :class:`pymongo.errors.InvalidId` -> :class:`bson.errors.InvalidId` The above exceptions now inherit from :class:`bson.errors.BSONError` rather than :class:`pymongo.errors.PyMongoError`. .. note:: All of the renamed modules and exceptions above have aliases created with the old names, so these changes should not break existing code. The old names will eventually be deprecated and then removed, so users should begin migrating towards the new names now. .. warning:: The change to the exception hierarchy mentioned above is possibly breaking. If your code is catching :class:`~pymongo.errors.PyMongoError`, then the exceptions raised by :mod:`bson` will not be caught, even though they would have been caught previously. Before upgrading, it is recommended that users check for any cases like this. - the C extension now shares buffer.c/h with the Ruby driver - :mod:`bson` no longer raises :class:`~pymongo.errors.InvalidName`, all occurrences have been replaced with :class:`~bson.errors.InvalidDocument`. - renamed :meth:`bson._to_dicts` to :meth:`~bson.decode_all`. - renamed :meth:`~bson.BSON.from_dict` to :meth:`~bson.BSON.encode` and :meth:`~bson.BSON.to_dict` to :meth:`~bson.BSON.decode`. - added :meth:`~pymongo.cursor.Cursor.batch_size`. - allow updating (some) file metadata after a :class:`~gridfs.grid_file.GridIn` instance has been closed. - performance improvements for reading from GridFS. - special cased slice with the same start and stop to return an empty cursor. - allow writing :class:`unicode` to GridFS if an :attr:`encoding` attribute has been specified for the file. - added :meth:`gridfs.GridFS.get_version`. - scope variables for :class:`~bson.code.Code` can now be specified as keyword arguments. - added :meth:`~gridfs.grid_file.GridOut.readline` to :class:`~gridfs.grid_file.GridOut`. - make a best effort to transparently auto-reconnect if a :class:`~pymongo.connection.Connection` has been idle for a while. - added :meth:`~pymongo.database.SystemJS.list` to :class:`~pymongo.database.SystemJS`. - added `file_document` argument to :meth:`~gridfs.grid_file.GridOut` to allow initializing from an existing file document. - raise :class:`~pymongo.errors.TimeoutError` even if the ``getLastError`` command was run manually and not through "safe" mode. - added :class:`uuid` support to :mod:`~bson.json_util`. Changes in Version 1.8.1 ------------------------ - fixed a typo in the C extension that could cause safe-mode operations to report a failure (:class:`SystemError`) even when none occurred. - added a :meth:`__ne__` implementation to any class where we define :meth:`__eq__`. Changes in Version 1.8 ---------------------- Version 1.8 adds support for connecting to replica sets, specifying per-operation values for `w` and `wtimeout`, and decoding to timezone-aware datetimes. - fixed a reference leak in the C extension when decoding a :class:`~bson.dbref.DBRef`. - added support for `w`, `wtimeout`, and `fsync` (and any other options for `getLastError`) to "safe mode" operations. - added :attr:`~pymongo.connection.Connection.nodes` property. - added a maximum pool size of 10 sockets. - added support for replica sets. - DEPRECATED :meth:`~pymongo.connection.Connection.from_uri` and :meth:`~pymongo.connection.Connection.paired`, both are supplanted by extended functionality in :meth:`~pymongo.connection.Connection`. - added tz aware support for datetimes in :class:`~bson.objectid.ObjectId`, :class:`~bson.timestamp.Timestamp` and :mod:`~bson.json_util` methods. - added :meth:`~pymongo.collection.Collection.drop` helper. - reuse the socket used for finding the master when a :class:`~pymongo.connection.Connection` is first created. - added support for :class:`~bson.min_key.MinKey`, :class:`~bson.max_key.MaxKey` and :class:`~bson.timestamp.Timestamp` to :mod:`~bson.json_util`. - added support for decoding datetimes as aware (UTC) - it is highly recommended to enable this by setting the `tz_aware` parameter to :meth:`~pymongo.connection.Connection` to ``True``. - added `network_timeout` option for individual calls to :meth:`~pymongo.collection.Collection.find` and :meth:`~pymongo.collection.Collection.find_one`. - added :meth:`~gridfs.GridFS.exists` to check if a file exists in GridFS. - added support for additional keys in :class:`~bson.dbref.DBRef` instances. - added :attr:`~pymongo.errors.OperationFailure.code` attribute to :class:`~pymongo.errors.OperationFailure` exceptions. - fixed serialization of int and float subclasses in the C extension. Changes in Version 1.7 ---------------------- Version 1.7 is a recommended upgrade for all PyMongo users. The full release notes are below, and some more in depth discussion of the highlights is `here `_. - no longer attempt to build the C extension on big-endian systems. - added :class:`~bson.min_key.MinKey` and :class:`~bson.max_key.MaxKey`. - use unsigned for :class:`~bson.timestamp.Timestamp` in BSON encoder/decoder. - support ``True`` as ``"ok"`` in command responses, in addition to ``1.0`` - necessary for server versions **>= 1.5.X** - BREAKING change to :meth:`~pymongo.collection.Collection.index_information` to add support for querying unique status and other index information. - added :attr:`~pymongo.connection.Connection.document_class`, to specify class for returned documents. - added `as_class` argument for :meth:`~pymongo.collection.Collection.find`, and in the BSON decoder. - added support for creating :class:`~bson.timestamp.Timestamp` instances using a :class:`~datetime.datetime`. - allow `dropTarget` argument for :class:`~pymongo.collection.Collection.rename`. - handle aware :class:`~datetime.datetime` instances, by converting to UTC. - added support for :class:`~pymongo.cursor.Cursor.max_scan`. - raise :class:`~gridfs.errors.FileExists` exception when creating a duplicate GridFS file. - use `y2038 `_ for time handling in the C extension - eliminates 2038 problems when extension is installed. - added `sort` parameter to :meth:`~pymongo.collection.Collection.find` - finalized deprecation of changes from versions **<= 1.4** - take any non-:class:`dict` as an ``"_id"`` query for :meth:`~pymongo.collection.Collection.find_one` or :meth:`~pymongo.collection.Collection.remove` - added ability to pass a :class:`dict` for `fields` argument to :meth:`~pymongo.collection.Collection.find` (supports ``"$slice"`` and field negation) - simplified code to find master, since paired setups don't always have a remote - fixed bug in C encoder for certain invalid types (like :class:`~pymongo.collection.Collection` instances). - don't transparently map ``"filename"`` key to :attr:`name` attribute for GridFS. Changes in Version 1.6 ---------------------- The biggest change in version 1.6 is a complete re-implementation of :mod:`gridfs` with a lot of improvements over the old implementation. There are many details and examples of using the new API in `this blog post `_. The old API has been removed in this version, so existing code will need to be modified before upgrading to 1.6. - fixed issue where connection pool was being shared across :class:`~pymongo.connection.Connection` instances. - more improvements to Python code caching in C extension - should improve behavior on mod_wsgi. - added :meth:`~bson.objectid.ObjectId.from_datetime`. - complete rewrite of :mod:`gridfs` support. - improvements to the :meth:`~pymongo.database.Database.command` API. - fixed :meth:`~pymongo.collection.Collection.drop_indexes` behavior on non-existent collections. - disallow empty bulk inserts. Changes in Version 1.5.2 ------------------------ - fixed response handling to ignore unknown response flags in queries. - handle server versions containing '-pre-'. Changes in Version 1.5.1 ------------------------ - added :data:`~gridfs.grid_file.GridFile._id` property for :class:`~gridfs.grid_file.GridFile` instances. - fix for making a :class:`~pymongo.connection.Connection` (with `slave_okay` set) directly to a slave in a replica pair. - accept kwargs for :meth:`~pymongo.collection.Collection.create_index` and :meth:`~pymongo.collection.Collection.ensure_index` to support all indexing options. - add :data:`pymongo.GEO2D` and support for geo indexing. - improvements to Python code caching in C extension - should improve behavior on mod_wsgi. Changes in Version 1.5 ---------------------- - added subtype constants to :mod:`~bson.binary` module. - DEPRECATED `options` argument to :meth:`~pymongo.collection.Collection` and :meth:`~pymongo.database.Database.create_collection` in favor of kwargs. - added :meth:`~pymongo.has_c` to check for C extension. - added :meth:`~pymongo.connection.Connection.copy_database`. - added :data:`~pymongo.cursor.Cursor.alive` to tell when a cursor might have more data to return (useful for tailable cursors). - added :class:`~bson.timestamp.Timestamp` to better support dealing with internal MongoDB timestamps. - added `name` argument for :meth:`~pymongo.collection.Collection.create_index` and :meth:`~pymongo.collection.Collection.ensure_index`. - fixed connection pooling w/ fork - :meth:`~pymongo.connection.Connection.paired` takes all kwargs that are allowed for :meth:`~pymongo.connection.Connection`. - :meth:`~pymongo.collection.Collection.insert` returns list for bulk inserts of size one. - fixed handling of :class:`datetime.datetime` instances in :mod:`~bson.json_util`. - added :meth:`~pymongo.connection.Connection.from_uri` to support MongoDB connection uri scheme. - fixed chunk number calculation when unaligned in :mod:`gridfs`. - :meth:`~pymongo.database.Database.command` takes a string for simple commands. - added :data:`~pymongo.database.Database.system_js` helper for dealing with server-side JS. - don't wrap queries containing ``"$query"`` (support manual use of ``"$min"``, etc.). - added :class:`~gridfs.errors.GridFSError` as base class for :mod:`gridfs` exceptions. Changes in Version 1.4 ---------------------- Perhaps the most important change in version 1.4 is that we have decided to **no longer support Python 2.3**. The most immediate reason for this is to allow some improvements to connection pooling. This will also allow us to use some new (as in Python 2.4 ;) idioms and will help begin the path towards supporting Python 3.0. If you need to use Python 2.3 you should consider using version 1.3 of this driver, although that will no longer be actively supported. Other changes: - move ``"_id"`` to front only for top-level documents (fixes some corner cases). - :meth:`~pymongo.collection.Collection.update` and :meth:`~pymongo.collection.Collection.remove` return the entire response to the *lastError* command when safe is ``True``. - completed removal of things that were deprecated in version 1.2 or earlier. - enforce that collection names do not contain the NULL byte. - fix to allow using UTF-8 collection names with the C extension. - added :class:`~pymongo.errors.PyMongoError` as base exception class for all :mod:`~pymongo.errors`. this changes the exception hierarchy somewhat, and is a BREAKING change if you depend on :class:`~pymongo.errors.ConnectionFailure` being a :class:`IOError` or :class:`~bson.errors.InvalidBSON` being a :class:`ValueError`, for example. - added :class:`~pymongo.errors.DuplicateKeyError` for calls to :meth:`~pymongo.collection.Collection.insert` or :meth:`~pymongo.collection.Collection.update` with `safe` set to ``True``. - removed :mod:`~pymongo.thread_util`. - added :meth:`~pymongo.database.Database.add_user` and :meth:`~pymongo.database.Database.remove_user` helpers. - fix for :meth:`~pymongo.database.Database.authenticate` when using non-UTF-8 names or passwords. - minor fixes for :class:`~pymongo.master_slave_connection.MasterSlaveConnection`. - clean up all cases where :class:`~pymongo.errors.ConnectionFailure` is raised. - simplification of connection pooling - makes driver ~2x faster for simple benchmarks. see :ref:`connection-pooling` for more information. - DEPRECATED `pool_size`, `auto_start_request` and `timeout` parameters to :class:`~pymongo.connection.Connection`. DEPRECATED :meth:`~pymongo.connection.Connection.start_request`. - use :meth:`socket.sendall`. - removed :meth:`~bson.son.SON.from_xml` as it was only being used for some internal testing - also eliminates dependency on :mod:`elementtree`. - implementation of :meth:`~pymongo.message.update` in C. - deprecate :meth:`~pymongo.database.Database._command` in favor of :meth:`~pymongo.database.Database.command`. - send all commands without wrapping as ``{"query": ...}``. - support string as `key` argument to :meth:`~pymongo.collection.Collection.group` (keyf) and run all groups as commands. - support for equality testing for :class:`~bson.code.Code` instances. - allow the NULL byte in strings and disallow it in key names or regex patterns Changes in Version 1.3 ---------------------- - DEPRECATED running :meth:`~pymongo.collection.Collection.group` as :meth:`~pymongo.database.Database.eval`, also changed default for :meth:`~pymongo.collection.Collection.group` to running as a command - remove :meth:`pymongo.cursor.Cursor.__len__`, which was deprecated in 1.1.1 - needed to do this aggressively due to it's presence breaking **Django** template *for* loops - DEPRECATED :meth:`~pymongo.connection.Connection.host`, :meth:`~pymongo.connection.Connection.port`, :meth:`~pymongo.database.Database.connection`, :meth:`~pymongo.database.Database.name`, :meth:`~pymongo.collection.Collection.database`, :meth:`~pymongo.collection.Collection.name` and :meth:`~pymongo.collection.Collection.full_name` in favor of :attr:`~pymongo.connection.Connection.host`, :attr:`~pymongo.connection.Connection.port`, :attr:`~pymongo.database.Database.connection`, :attr:`~pymongo.database.Database.name`, :attr:`~pymongo.collection.Collection.database`, :attr:`~pymongo.collection.Collection.name` and :attr:`~pymongo.collection.Collection.full_name`, respectively. The deprecation schedule for this change will probably be faster than usual, as it carries some performance implications. - added :meth:`~pymongo.connection.Connection.disconnect` Changes in Version 1.2.1 ------------------------ - added :doc:`changelog` to docs - added ``setup.py doc --test`` to run doctests for tutorial, examples - moved most examples to Sphinx docs (and remove from *examples/* directory) - raise :class:`~bson.errors.InvalidId` instead of :class:`TypeError` when passing a 24 character string to :class:`~bson.objectid.ObjectId` that contains non-hexadecimal characters - allow :class:`unicode` instances for :class:`~bson.objectid.ObjectId` init Changes in Version 1.2 ---------------------- - `spec` parameter for :meth:`~pymongo.collection.Collection.remove` is now optional to allow for deleting all documents in a :class:`~pymongo.collection.Collection` - always wrap queries with ``{query: ...}`` even when no special options - get around some issues with queries on fields named ``query`` - enforce 4MB document limit on the client side - added :meth:`~pymongo.collection.Collection.map_reduce` helper - see :doc:`example ` - added :meth:`~pymongo.cursor.Cursor.distinct` method on :class:`~pymongo.cursor.Cursor` instances to allow distinct with queries - fix for :meth:`~pymongo.cursor.Cursor.__getitem__` after :meth:`~pymongo.cursor.Cursor.skip` - allow any UTF-8 string in :class:`~bson.BSON` encoder, not just ASCII subset - added :attr:`~bson.objectid.ObjectId.generation_time` - removed support for legacy :class:`~bson.objectid.ObjectId` format - pretty sure this was never used, and is just confusing - DEPRECATED :meth:`~bson.objectid.ObjectId.url_encode` and :meth:`~bson.objectid.ObjectId.url_decode` in favor of :meth:`str` and :meth:`~bson.objectid.ObjectId`, respectively - allow *oplog.$main* as a valid collection name - some minor fixes for installation process - added support for datetime and regex in :mod:`~bson.json_util` Changes in Version 1.1.2 ------------------------ - improvements to :meth:`~pymongo.collection.Collection.insert` speed (using C for insert message creation) - use random number for request_id - fix some race conditions with :class:`~pymongo.errors.AutoReconnect` Changes in Version 1.1.1 ------------------------ - added `multi` parameter for :meth:`~pymongo.collection.Collection.update` - fix unicode regex patterns with C extension - added :meth:`~pymongo.collection.Collection.distinct` - added `database` support for :class:`~bson.dbref.DBRef` - added :mod:`~bson.json_util` with helpers for encoding / decoding special types to JSON - DEPRECATED :meth:`pymongo.cursor.Cursor.__len__` in favor of :meth:`~pymongo.cursor.Cursor.count` with `with_limit_and_skip` set to ``True`` due to performance regression - switch documentation to Sphinx Changes in Version 1.1 ---------------------- - added :meth:`__hash__` for :class:`~bson.dbref.DBRef` and :class:`~bson.objectid.ObjectId` - bulk :meth:`~pymongo.collection.Collection.insert` works with any iterable - fix :class:`~bson.objectid.ObjectId` generation when using :mod:`multiprocessing` - added :attr:`~pymongo.cursor.Cursor.collection` - added `network_timeout` parameter for :meth:`~pymongo.connection.Connection` - DEPRECATED `slave_okay` parameter for individual queries - fix for `safe` mode when multi-threaded - added `safe` parameter for :meth:`~pymongo.collection.Collection.remove` - added `tailable` parameter for :meth:`~pymongo.collection.Collection.find` Changes in Version 1.0 ---------------------- - fixes for :class:`~pymongo.master_slave_connection.MasterSlaveConnection` - added `finalize` parameter for :meth:`~pymongo.collection.Collection.group` - improvements to :meth:`~pymongo.collection.Collection.insert` speed - improvements to :mod:`gridfs` speed - added :meth:`~pymongo.cursor.Cursor.__getitem__` and :meth:`~pymongo.cursor.Cursor.__len__` for :class:`~pymongo.cursor.Cursor` instances Changes in Version 0.16 ----------------------- - support for encoding/decoding :class:`uuid.UUID` instances - fix for :meth:`~pymongo.cursor.Cursor.explain` with limits Changes in Version 0.15.2 ------------------------- - documentation changes only Changes in Version 0.15.1 ------------------------- - various performance improvements - API CHANGE no longer need to specify direction for :meth:`~pymongo.collection.Collection.create_index` and :meth:`~pymongo.collection.Collection.ensure_index` when indexing a single key - support for encoding :class:`tuple` instances as :class:`list` instances Changes in Version 0.15 ----------------------- - fix string representation of :class:`~bson.objectid.ObjectId` instances - added `timeout` parameter for :meth:`~pymongo.collection.Collection.find` - allow scope for `reduce` function in :meth:`~pymongo.collection.Collection.group` Changes in Version 0.14.2 ------------------------- - minor bugfixes Changes in Version 0.14.1 ------------------------- - :meth:`~gridfs.grid_file.GridFile.seek` and :meth:`~gridfs.grid_file.GridFile.tell` for (read mode) :class:`~gridfs.grid_file.GridFile` instances Changes in Version 0.14 ----------------------- - support for long in :class:`~bson.BSON` - added :meth:`~pymongo.collection.Collection.rename` - added `snapshot` parameter for :meth:`~pymongo.collection.Collection.find` Changes in Version 0.13 ----------------------- - better :class:`~pymongo.master_slave_connection.MasterSlaveConnection` support - API CHANGE :meth:`~pymongo.collection.Collection.insert` and :meth:`~pymongo.collection.Collection.save` both return inserted ``_id`` - DEPRECATED passing an index name to :meth:`~pymongo.cursor.Cursor.hint` Changes in Version 0.12 ----------------------- - improved :class:`~bson.objectid.ObjectId` generation - added :class:`~pymongo.errors.AutoReconnect` exception for when reconnection is possible - make :mod:`gridfs` thread-safe - fix for :mod:`gridfs` with non :class:`~bson.objectid.ObjectId` ``_id`` Changes in Version 0.11.3 ------------------------- - don't allow NULL bytes in string encoder - fixes for Python 2.3 Changes in Version 0.11.2 ------------------------- - PEP 8 - updates for :meth:`~pymongo.collection.Collection.group` - VS build Changes in Version 0.11.1 ------------------------- - fix for connection pooling under Python 2.5 Changes in Version 0.11 ----------------------- - better build failure detection - driver support for selecting fields in sub-documents - disallow insertion of invalid key names - added `timeout` parameter for :meth:`~pymongo.connection.Connection` Changes in Version 0.10.3 ------------------------- - fix bug with large :meth:`~pymongo.cursor.Cursor.limit` - better exception when modules get reloaded out from underneath the C extension - better exception messages when calling a :class:`~pymongo.collection.Collection` or :class:`~pymongo.database.Database` instance Changes in Version 0.10.2 ------------------------- - support subclasses of :class:`dict` in C encoder Changes in Version 0.10.1 ------------------------- - alias :class:`~pymongo.connection.Connection` as :attr:`pymongo.Connection` - raise an exception rather than silently overflowing in encoder Changes in Version 0.10 ----------------------- - added :meth:`~pymongo.collection.Collection.ensure_index` Changes in Version 0.9.7 ------------------------ - allow sub-collections of *$cmd* as valid :class:`~pymongo.collection.Collection` names - add version as :attr:`pymongo.version` - add ``--no_ext`` command line option to *setup.py* .. toctree:: :hidden: python3 examples/gevent pymongo-3.6.1/doc/python3.rst0000644000076600000240000001071613245621354016350 0ustar shanestaff00000000000000Python 3 FAQ ============ .. contents:: What Python 3 versions are supported? ------------------------------------- PyMongo supports CPython 3.4+ and PyPy3. Are there any PyMongo behavior changes with Python 3? ----------------------------------------------------- Only one intentional change. Instances of :class:`bytes` are encoded as BSON type 5 (Binary data) with subtype 0. In Python 3 they are decoded back to :class:`bytes`. In Python 2 they are decoded to :class:`~bson.binary.Binary` with subtype 0. For example, let's insert a :class:`bytes` instance using Python 3 then read it back. Notice the byte string is decoded back to :class:`bytes`:: Python 3.6.1 (v3.6.1:69c0db5050, Mar 21 2017, 01:21:04) [GCC 4.9.3] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import pymongo >>> c = pymongo.MongoClient() >>> c.test.bintest.insert_one({'binary': b'this is a byte string'}).inserted_id ObjectId('4f9086b1fba5222021000000') >>> c.test.bintest.find_one() {'binary': b'this is a byte string', '_id': ObjectId('4f9086b1fba5222021000000')} Now retrieve the same document in Python 2. Notice the byte string is decoded to :class:`~bson.binary.Binary`:: Python 2.7.6 (default, Feb 26 2014, 10:36:22) [GCC 4.7.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pymongo >>> c = pymongo.MongoClient() >>> c.test.bintest.find_one() {u'binary': Binary('this is a byte string', 0), u'_id': ObjectId('4f9086b1fba5222021000000')} There is a similar change in behavior in parsing JSON binary with subtype 0. In Python 3 they are decoded into :class:`bytes`. In Python 2 they are decoded to :class:`~bson.binary.Binary` with subtype 0. For example, let's decode a JSON binary subtype 0 using Python 3. Notice the byte string is decoded to :class:`bytes`:: Python 3.6.1 (v3.6.1:69c0db5050, Mar 21 2017, 01:21:04) [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> from bson.json_util import loads >>> loads('{"b": {"$binary": "dGhpcyBpcyBhIGJ5dGUgc3RyaW5n", "$type": "00"}}') {'b': b'this is a byte string'} Now decode the same JSON in Python 2 . Notice the byte string is decoded to :class:`~bson.binary.Binary`:: Python 2.7.10 (default, Feb 7 2017, 00:08:15) [GCC 4.2.1 Compatible Apple LLVM 8.0.0 (clang-800.0.34)] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> from bson.json_util import loads >>> loads('{"b": {"$binary": "dGhpcyBpcyBhIGJ5dGUgc3RyaW5n", "$type": "00"}}') {u'b': Binary('this is a byte string', 0)} Why can't I share pickled ObjectIds between some versions of Python 2 and 3? ---------------------------------------------------------------------------- Instances of :class:`~bson.objectid.ObjectId` pickled using Python 2 can always be unpickled using Python 3. If you pickled an ObjectId using Python 2 and want to unpickle it using Python 3 you must pass ``encoding='latin-1'`` to pickle.loads:: Python 2.7.6 (default, Feb 26 2014, 10:36:22) [GCC 4.7.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> from bson.objectid import ObjectId >>> oid = ObjectId() >>> oid ObjectId('4f919ba2fba5225b84000000') >>> pickle.dumps(oid) 'ccopy_reg\n_reconstructor\np0\n(cbson.objectid\...' Python 3.6.1 (v3.6.1:69c0db5050, Mar 21 2017, 01:21:04) [GCC 4.9.3] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> pickle.loads(b'ccopy_reg\n_reconstructor\np0\n(cbson.objectid\...', encoding='latin-1') ObjectId('4f919ba2fba5225b84000000') If you need to pickle ObjectIds using Python 3 and unpickle them using Python 2 you must use ``protocol <= 2``:: Python 3.6.1 (v3.6.1:69c0db5050, Mar 21 2017, 01:21:04) [GCC 4.9.3] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> from bson.objectid import ObjectId >>> oid = ObjectId() >>> oid ObjectId('4f96f20c430ee6bd06000000') >>> pickle.dumps(oid, protocol=2) b'\x80\x02cbson.objectid\nObjectId\nq\x00)\x81q\x01c_codecs\nencode\...' Python 2.6.9 (unknown, Feb 26 2014, 12:39:10) [GCC 4.7.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> pickle.loads('\x80\x02cbson.objectid\nObjectId\nq\x00)\x81q\x01c_codecs\nencode\...') ObjectId('4f96f20c430ee6bd06000000') pymongo-3.6.1/doc/faq.rst0000644000076600000240000005322013245621354015510 0ustar shanestaff00000000000000Frequently Asked Questions ========================== .. contents:: Is PyMongo thread-safe? ----------------------- PyMongo is thread-safe and provides built-in connection pooling for threaded applications. .. _pymongo-fork-safe: Is PyMongo fork-safe? --------------------- PyMongo is not fork-safe. Care must be taken when using instances of :class:`~pymongo.mongo_client.MongoClient` with ``fork()``. Specifically, instances of MongoClient must not be copied from a parent process to a child process. Instead, the parent process and each child process must create their own instances of MongoClient. Instances of MongoClient copied from the parent process have a high probability of deadlock in the child process due to the inherent incompatibilities between ``fork()``, threads, and locks described :ref:`below `. PyMongo will attempt to issue a warning if there is a chance of this deadlock occurring. .. _pymongo-fork-safe-details: MongoClient spawns multiple threads to run background tasks such as monitoring connected servers. These threads share state that is protected by instances of :class:`~threading.Lock`, which are themselves `not fork-safe`_. The driver is therefore subject to the same limitations as any other multithreaded code that uses :class:`~threading.Lock` (and mutexes in general). One of these limitations is that the locks become useless after ``fork()``. During the fork, all locks are copied over to the child process in the same state as they were in the parent: if they were locked, the copied locks are also locked. The child created by ``fork()`` only has one thread, so any locks that were taken out by other threads in the parent will never be released in the child. The next time the child process attempts to acquire one of these locks, deadlock occurs. For a long but interesting read about the problems of Python locks in multithreaded contexts with ``fork()``, see http://bugs.python.org/issue6721. .. _not fork-safe: http://bugs.python.org/issue6721 .. _connection-pooling: How does connection pooling work in PyMongo? -------------------------------------------- Every :class:`~pymongo.mongo_client.MongoClient` instance has a built-in connection pool per server in your MongoDB topology. These pools open sockets on demand to support the number of concurrent MongoDB operations that your multi-threaded application requires. There is no thread-affinity for sockets. The size of each connection pool is capped at ``maxPoolSize``, which defaults to 100. If there are ``maxPoolSize`` connections to a server and all are in use, the next request to that server will wait until one of the connections becomes available. The client instance opens one additional socket per server in your MongoDB topology for monitoring the server's state. For example, a client connected to a 3-node replica set opens 3 monitoring sockets. It also opens as many sockets as needed to support a multi-threaded application's concurrent operations on each server, up to ``maxPoolSize``. With a ``maxPoolSize`` of 100, if the application only uses the primary (the default), then only the primary connection pool grows and the total connections is at most 103. If the application uses a :class:`~pymongo.read_preferences.ReadPreference` to query the secondaries, their pools also grow and the total connections can reach 303. It is possible to set the minimum number of concurrent connections to each server with ``minPoolSize``, which defaults to 0. The connection pool will be initialized with this number of sockets. If sockets are closed due to any network errors, causing the total number of sockets (both in use and idle) to drop below the minimum, more sockets are opened until the minimum is reached. The maximum number of milliseconds that a connection can remain idle in the pool before being removed and replaced can be set with ``maxIdleTime``, which defaults to `None` (no limit). The default configuration for a :class:`~pymongo.mongo_client.MongoClient` works for most applications:: client = MongoClient(host, port) Create this client **once** for each process, and reuse it for all operations. It is a common mistake to create a new client for each request, which is very inefficient. To support extremely high numbers of concurrent MongoDB operations within one process, increase ``maxPoolSize``:: client = MongoClient(host, port, maxPoolSize=200) ... or make it unbounded:: client = MongoClient(host, port, maxPoolSize=None) By default, any number of threads are allowed to wait for sockets to become available, and they can wait any length of time. Override ``waitQueueMultiple`` to cap the number of waiting threads. E.g., to keep the number of waiters less than or equal to 500:: client = MongoClient(host, port, maxPoolSize=50, waitQueueMultiple=10) When 500 threads are waiting for a socket, the 501st that needs a socket raises :exc:`~pymongo.errors.ExceededMaxWaiters`. Use this option to bound the amount of queueing in your application during a load spike, at the cost of additional exceptions. Once the pool reaches its max size, additional threads are allowed to wait indefinitely for sockets to become available, unless you set ``waitQueueTimeoutMS``:: client = MongoClient(host, port, waitQueueTimeoutMS=100) A thread that waits more than 100ms (in this example) for a socket raises :exc:`~pymongo.errors.ConnectionFailure`. Use this option if it is more important to bound the duration of operations during a load spike than it is to complete every operation. When :meth:`~pymongo.mongo_client.MongoClient.close` is called by any thread, all idle sockets are closed, and all sockets that are in use will be closed as they are returned to the pool. Does PyMongo support Python 3? ------------------------------ PyMongo supports CPython 3.4+ and PyPy3. See the :doc:`python3` for details. Does PyMongo support asynchronous frameworks like Gevent, asyncio, Tornado, or Twisted? --------------------------------------------------------------------------------------- PyMongo fully supports :doc:`Gevent `. To use MongoDB with `asyncio `_ or `Tornado `_, see the `Motor `_ project. For `Twisted `_, see `TxMongo `_. Its stated mission is to keep feature parity with PyMongo. .. _writes-and-ids: Why does PyMongo add an _id field to all of my documents? --------------------------------------------------------- When a document is inserted to MongoDB using :meth:`~pymongo.collection.Collection.insert_one`, :meth:`~pymongo.collection.Collection.insert_many`, or :meth:`~pymongo.collection.Collection.bulk_write`, and that document does not include an ``_id`` field, PyMongo automatically adds one for you, set to an instance of :class:`~bson.objectid.ObjectId`. For example:: >>> my_doc = {'x': 1} >>> collection.insert_one(my_doc) >>> my_doc {'x': 1, '_id': ObjectId('560db337fba522189f171720')} Users often discover this behavior when calling :meth:`~pymongo.collection.Collection.insert_many` with a list of references to a single document raises :exc:`~pymongo.errors.BulkWriteError`. Several Python idioms lead to this pitfall:: >>> doc = {} >>> collection.insert_many(doc for _ in range(10)) Traceback (most recent call last): ... pymongo.errors.BulkWriteError: batch op errors occurred >>> doc {'_id': ObjectId('560f171cfba52279f0b0da0c')} >>> docs = [{}] >>> collection.insert_many(docs * 10) Traceback (most recent call last): ... pymongo.errors.BulkWriteError: batch op errors occurred >>> docs [{'_id': ObjectId('560f1933fba52279f0b0da0e')}] PyMongo adds an ``_id`` field in this manner for a few reasons: - All MongoDB documents are required to have an ``_id`` field. - If PyMongo were to insert a document without an ``_id`` MongoDB would add one itself, but it would not report the value back to PyMongo. - Copying the document to insert before adding the ``_id`` field would be prohibitively expensive for most high write volume applications. If you don't want PyMongo to add an ``_id`` to your documents, insert only documents that already have an ``_id`` field, added by your application. Key order in subdocuments -- why does my query work in the shell but not PyMongo? --------------------------------------------------------------------------------- .. testsetup:: key-order from bson.son import SON from pymongo.mongo_client import MongoClient collection = MongoClient().test.collection collection.drop() collection.insert_one({'_id': 1.0, 'subdocument': SON([('b', 1.0), ('a', 1.0)])}) The key-value pairs in a BSON document can have any order (except that ``_id`` is always first). The mongo shell preserves key order when reading and writing data. Observe that "b" comes before "a" when we create the document and when it is displayed: .. code-block:: javascript > // mongo shell. > db.collection.insert( { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } ) WriteResult({ "nInserted" : 1 }) > db.collection.find() { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } PyMongo represents BSON documents as Python dicts by default, and the order of keys in dicts is not defined. That is, a dict declared with the "a" key first is the same, to Python, as one with "b" first: >>> print({'a': 1.0, 'b': 1.0}) {'a': 1.0, 'b': 1.0} >>> print({'b': 1.0, 'a': 1.0}) {'a': 1.0, 'b': 1.0} Therefore, Python dicts are not guaranteed to show keys in the order they are stored in BSON. Here, "a" is shown before "b": >>> print(collection.find_one()) {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}} To preserve order when reading BSON, use the :class:`~bson.son.SON` class, which is a dict that remembers its key order. First, get a handle to the collection, configured to use :class:`~bson.son.SON` instead of dict: .. doctest:: key-order :options: +NORMALIZE_WHITESPACE >>> from bson import CodecOptions, SON >>> opts = CodecOptions(document_class=SON) >>> opts CodecOptions(document_class=, tz_aware=False, uuid_representation=PYTHON_LEGACY, unicode_decode_error_handler='strict', tzinfo=None) >>> collection_son = collection.with_options(codec_options=opts) Now, documents and subdocuments in query results are represented with :class:`~bson.son.SON` objects: .. doctest:: key-order >>> print(collection_son.find_one()) SON([(u'_id', 1.0), (u'subdocument', SON([(u'b', 1.0), (u'a', 1.0)]))]) The subdocument's actual storage layout is now visible: "b" is before "a". Because a dict's key order is not defined, you cannot predict how it will be serialized **to** BSON. But MongoDB considers subdocuments equal only if their keys have the same order. So if you use a dict to query on a subdocument it may not match: >>> collection.find_one({'subdocument': {'a': 1.0, 'b': 1.0}}) is None True Swapping the key order in your query makes no difference: >>> collection.find_one({'subdocument': {'b': 1.0, 'a': 1.0}}) is None True ... because, as we saw above, Python considers the two dicts the same. There are two solutions. First, you can match the subdocument field-by-field: >>> collection.find_one({'subdocument.a': 1.0, ... 'subdocument.b': 1.0}) {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}} The query matches any subdocument with an "a" of 1.0 and a "b" of 1.0, regardless of the order you specify them in Python or the order they are stored in BSON. Additionally, this query now matches subdocuments with additional keys besides "a" and "b", whereas the previous query required an exact match. The second solution is to use a :class:`~bson.son.SON` to specify the key order: >>> query = {'subdocument': SON([('b', 1.0), ('a', 1.0)])} >>> collection.find_one(query) {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}} The key order you use when you create a :class:`~bson.son.SON` is preserved when it is serialized to BSON and used as a query. Thus you can create a subdocument that exactly matches the subdocument in the collection. .. seealso:: `MongoDB Manual entry on subdocument matching `_. What does *CursorNotFound* cursor id not valid at server mean? -------------------------------------------------------------- Cursors in MongoDB can timeout on the server if they've been open for a long time without any operations being performed on them. This can lead to an :class:`~pymongo.errors.CursorNotFound` exception being raised when attempting to iterate the cursor. How do I change the timeout value for cursors? ---------------------------------------------- MongoDB doesn't support custom timeouts for cursors, but cursor timeouts can be turned off entirely. Pass ``no_cursor_timeout=True`` to :meth:`~pymongo.collection.Collection.find`. How can I store :mod:`decimal.Decimal` instances? ------------------------------------------------- PyMongo >= 3.4 supports the Decimal128 BSON type introduced in MongoDB 3.4. See :mod:`~bson.decimal128` for more information. MongoDB <= 3.2 only supports IEEE 754 floating points - the same as the Python float type. The only way PyMongo could store Decimal instances to these versions of MongoDB would be to convert them to this standard, so you'd really only be storing floats anyway - we force users to do this conversion explicitly so that they are aware that it is happening. I'm saving ``9.99`` but when I query my document contains ``9.9900000000000002`` - what's going on here? -------------------------------------------------------------------------------------------------------- The database representation is ``9.99`` as an IEEE floating point (which is common to MongoDB and Python as well as most other modern languages). The problem is that ``9.99`` cannot be represented exactly with a double precision floating point - this is true in some versions of Python as well: >>> 9.99 9.9900000000000002 The result that you get when you save ``9.99`` with PyMongo is exactly the same as the result you'd get saving it with the JavaScript shell or any of the other languages (and as the data you're working with when you type ``9.99`` into a Python program). Can you add attribute style access for documents? ------------------------------------------------- This request has come up a number of times but we've decided not to implement anything like this. The relevant `jira case `_ has some information about the decision, but here is a brief summary: 1. This will pollute the attribute namespace for documents, so could lead to subtle bugs / confusing errors when using a key with the same name as a dictionary method. 2. The only reason we even use SON objects instead of regular dictionaries is to maintain key ordering, since the server requires this for certain operations. So we're hesitant to needlessly complicate SON (at some point it's hypothetically possible we might want to revert back to using dictionaries alone, without breaking backwards compatibility for everyone). 3. It's easy (and Pythonic) for new users to deal with documents, since they behave just like dictionaries. If we start changing their behavior it adds a barrier to entry for new users - another class to learn. What is the correct way to handle time zones with PyMongo? ---------------------------------------------------------- See :doc:`examples/datetimes` for examples on how to handle :class:`~datetime.datetime` objects correctly. How can I save a :mod:`datetime.date` instance? ----------------------------------------------- PyMongo doesn't support saving :mod:`datetime.date` instances, since there is no BSON type for dates without times. Rather than having the driver enforce a convention for converting :mod:`datetime.date` instances to :mod:`datetime.datetime` instances for you, any conversion should be performed in your client code. .. _web-application-querying-by-objectid: When I query for a document by ObjectId in my web application I get no result ----------------------------------------------------------------------------- It's common in web applications to encode documents' ObjectIds in URLs, like:: "/posts/50b3bda58a02fb9a84d8991e" Your web framework will pass the ObjectId portion of the URL to your request handler as a string, so it must be converted to :class:`~bson.objectid.ObjectId` before it is passed to :meth:`~pymongo.collection.Collection.find_one`. It is a common mistake to forget to do this conversion. Here's how to do it correctly in Flask_ (other web frameworks are similar):: from pymongo import MongoClient from bson.objectid import ObjectId from flask import Flask, render_template client = MongoClient() app = Flask(__name__) @app.route("/posts/<_id>") def show_post(_id): # NOTE!: converting _id from string to ObjectId before passing to find_one post = client.db.posts.find_one({'_id': ObjectId(_id)}) return render_template('post.html', post=post) if __name__ == "__main__": app.run() .. _Flask: http://flask.pocoo.org/ .. seealso:: :ref:`querying-by-objectid` How can I use PyMongo from Django? ---------------------------------- `Django `_ is a popular Python web framework. Django includes an ORM, :mod:`django.db`. Currently, there's no official MongoDB backend for Django. `django-mongodb-engine `_ is an unofficial MongoDB backend that supports Django aggregations, (atomic) updates, embedded objects, Map/Reduce and GridFS. It allows you to use most of Django's built-in features, including the ORM, admin, authentication, site and session frameworks and caching. However, it's easy to use MongoDB (and PyMongo) from Django without using a Django backend. Certain features of Django that require :mod:`django.db` (admin, authentication and sessions) will not work using just MongoDB, but most of what Django provides can still be used. One project which should make working with MongoDB and Django easier is `mango `_. Mango is a set of MongoDB backends for Django sessions and authentication (bypassing :mod:`django.db` entirely). .. _using-with-mod-wsgi: Does PyMongo work with **mod_wsgi**? ------------------------------------ Yes. See the configuration guide for :ref:`pymongo-and-mod_wsgi`. How can I use something like Python's :mod:`json` module to encode my documents to JSON? ---------------------------------------------------------------------------------------- :mod:`~bson.json_util` is PyMongo's built in, flexible tool for using Python's :mod:`json` module with BSON documents and `MongoDB Extended JSON `_. The :mod:`json` module won't work out of the box with all documents from PyMongo as PyMongo supports some special types (like :class:`~bson.objectid.ObjectId` and :class:`~bson.dbref.DBRef`) that are not supported in JSON. `python-bsonjs `_ is a fast BSON to MongoDB Extended JSON converter built on top of `libbson `_. `python-bsonjs` does not depend on PyMongo and can offer a nice performance improvement over :mod:`~bson.json_util`. `python-bsonjs` works best with PyMongo when using :class:`~bson.raw_bson.RawBSONDocument`. Why do I get OverflowError decoding dates stored by another language's driver? ------------------------------------------------------------------------------ PyMongo decodes BSON datetime values to instances of Python's :class:`datetime.datetime`. Instances of :class:`datetime.datetime` are limited to years between :data:`datetime.MINYEAR` (usually 1) and :data:`datetime.MAXYEAR` (usually 9999). Some MongoDB drivers (e.g. the PHP driver) can store BSON datetimes with year values far outside those supported by :class:`datetime.datetime`. There are a few ways to work around this issue. One option is to filter out documents with values outside of the range supported by :class:`datetime.datetime`:: >>> from datetime import datetime >>> coll = client.test.dates >>> cur = coll.find({'dt': {'$gte': datetime.min, '$lte': datetime.max}}) Another option, assuming you don't need the datetime field, is to filter out just that field:: >>> cur = coll.find({}, projection={'dt': False}) .. _multiprocessing: Using PyMongo with Multiprocessing ---------------------------------- On Unix systems the multiprocessing module spawns processes using ``fork()``. Care must be taken when using instances of :class:`~pymongo.mongo_client.MongoClient` with ``fork()``. Specifically, instances of MongoClient must not be copied from a parent process to a child process. Instead, the parent process and each child process must create their own instances of MongoClient. For example:: # Each process creates its own instance of MongoClient. def func(): db = pymongo.MongoClient().mydb # Do something with db. proc = multiprocessing.Process(target=func) proc.start() **Never do this**:: client = pymongo.MongoClient() # Each child process attempts to copy a global MongoClient # created in the parent process. Never do this. def func(): db = client.mydb # Do something with db. proc = multiprocessing.Process(target=func) proc.start() Instances of MongoClient copied from the parent process have a high probability of deadlock in the child process due to :ref:`inherent incompatibilities between fork(), threads, and locks `. PyMongo will attempt to issue a warning if there is a chance of this deadlock occurring. .. seealso:: :ref:`pymongo-fork-safe` pymongo-3.6.1/setup.cfg0000644000076600000240000000015013246104133015245 0ustar shanestaff00000000000000[nosetests] py3where = build with-xunit = 1 [egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 pymongo-3.6.1/ez_setup.py0000644000076600000240000003037113245617773015666 0ustar shanestaff00000000000000#!/usr/bin/env python """ Setuptools bootstrapping installer. Maintained at https://github.com/pypa/setuptools/tree/bootstrap. Run this script to install or upgrade setuptools. This method is DEPRECATED. Check https://github.com/pypa/setuptools/issues/581 for more details. """ import os import shutil import sys import tempfile import zipfile import optparse import subprocess import platform import textwrap import contextlib from distutils import log try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen try: from site import USER_SITE except ImportError: USER_SITE = None # 33.1.1 is the last version that supports setuptools self upgrade/installation. DEFAULT_VERSION = "33.1.1" DEFAULT_URL = "https://pypi.io/packages/source/s/setuptools/" DEFAULT_SAVE_DIR = os.curdir DEFAULT_DEPRECATION_MESSAGE = "ez_setup.py is deprecated and when using it setuptools will be pinned to {0} since it's the last version that supports setuptools self upgrade/installation, check https://github.com/pypa/setuptools/issues/581 for more info; use pip to install setuptools" MEANINGFUL_INVALID_ZIP_ERR_MSG = 'Maybe {0} is corrupted, delete it and try again.' log.warn(DEFAULT_DEPRECATION_MESSAGE.format(DEFAULT_VERSION)) def _python_cmd(*args): """ Execute a command. Return True if the command succeeded. """ args = (sys.executable,) + args return subprocess.call(args) == 0 def _install(archive_filename, install_args=()): """Install Setuptools.""" with archive_context(archive_filename): # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): log.warn('Something went wrong during the installation.') log.warn('See the error message above.') # exitcode will be 2 return 2 def _build_egg(egg, archive_filename, to_dir): """Build Setuptools egg.""" with archive_context(archive_filename): # building an egg log.warn('Building a Setuptools egg in %s', to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') class ContextualZipFile(zipfile.ZipFile): """Supplement ZipFile class to support context manager for Python 2.6.""" def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __new__(cls, *args, **kwargs): """Construct a ZipFile or ContextualZipFile as appropriate.""" if hasattr(zipfile.ZipFile, '__exit__'): return zipfile.ZipFile(*args, **kwargs) return super(ContextualZipFile, cls).__new__(cls) @contextlib.contextmanager def archive_context(filename): """ Unzip filename to a temporary directory, set to the cwd. The unzipped target is cleaned up after. """ tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) try: with ContextualZipFile(filename) as archive: archive.extractall() except zipfile.BadZipfile as err: if not err.args: err.args = ('', ) err.args = err.args + ( MEANINGFUL_INVALID_ZIP_ERR_MSG.format(filename), ) raise # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) yield finally: os.chdir(old_wd) shutil.rmtree(tmpdir) def _do_download(version, download_base, to_dir, download_delay): """Download Setuptools.""" py_desig = 'py{sys.version_info[0]}.{sys.version_info[1]}'.format(sys=sys) tp = 'setuptools-{version}-{py_desig}.egg' egg = os.path.join(to_dir, tp.format(**locals())) if not os.path.exists(egg): archive = download_setuptools(version, download_base, to_dir, download_delay) _build_egg(egg, archive, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: _unload_pkg_resources() import setuptools setuptools.bootstrap_install_from = egg def use_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=DEFAULT_SAVE_DIR, download_delay=15): """ Ensure that a setuptools version is installed. Return None. Raise SystemExit if the requested version or later cannot be installed. """ to_dir = os.path.abspath(to_dir) # prior to importing, capture the module state for # representative modules. rep_modules = 'pkg_resources', 'setuptools' imported = set(sys.modules).intersection(rep_modules) try: import pkg_resources pkg_resources.require("setuptools>=" + version) # a suitable version is already installed return except ImportError: # pkg_resources not available; setuptools is not installed; download pass except pkg_resources.DistributionNotFound: # no version of setuptools was found; allow download pass except pkg_resources.VersionConflict as VC_err: if imported: _conflict_bail(VC_err, version) # otherwise, unload pkg_resources to allow the downloaded version to # take precedence. del pkg_resources _unload_pkg_resources() return _do_download(version, download_base, to_dir, download_delay) def _conflict_bail(VC_err, version): """ Setuptools was imported prior to invocation, so it is unsafe to unload it. Bail out. """ conflict_tmpl = textwrap.dedent(""" The required version of setuptools (>={version}) is not available, and can't be installed while this script is running. Please install a more recent version first, using 'easy_install -U setuptools'. (Currently using {VC_err.args[0]!r}) """) msg = conflict_tmpl.format(**locals()) sys.stderr.write(msg) sys.exit(2) def _unload_pkg_resources(): sys.meta_path = [ importer for importer in sys.meta_path if importer.__class__.__module__ != 'pkg_resources.extern' ] del_modules = [ name for name in sys.modules if name.startswith('pkg_resources') ] for mod_name in del_modules: del sys.modules[mod_name] def _clean_check(cmd, target): """ Run the command to download target. If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) except subprocess.CalledProcessError: if os.access(target, os.F_OK): os.unlink(target) raise def download_file_powershell(url, target): """ Download the file at url to target using Powershell. Powershell will validate trust. Raise an exception if the command cannot complete. """ target = os.path.abspath(target) ps_cmd = ( "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " "[System.Net.CredentialCache]::DefaultCredentials; " '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' % locals() ) cmd = [ 'powershell', '-Command', ps_cmd, ] _clean_check(cmd, target) def has_powershell(): """Determine if Powershell is available.""" if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_powershell.viable = has_powershell def download_file_curl(url, target): cmd = ['curl', url, '--location', '--silent', '--output', target] _clean_check(cmd, target) def has_curl(): cmd = ['curl', '--version'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_curl.viable = has_curl def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) def has_wget(): cmd = ['wget', '--version'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_wget.viable = has_wget def download_file_insecure(url, target): """Use Python to download the file, without connection authentication.""" src = urlopen(url) try: # Read all the data in one block. data = src.read() finally: src.close() # Write all the data in one block to avoid creating a partial file. with open(target, "wb") as dst: dst.write(data) download_file_insecure.viable = lambda: True def get_best_downloader(): downloaders = ( download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, ) viable_downloaders = (dl for dl in downloaders if dl.viable()) return next(viable_downloaders, None) def download_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=DEFAULT_SAVE_DIR, delay=15, downloader_factory=get_best_downloader): """ Download setuptools from a specified location and return its filename. `version` should be a valid setuptools version number that is available as an sdist for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ # making sure we use the absolute path to_dir = os.path.abspath(to_dir) zip_name = "setuptools-%s.zip" % version url = download_base + zip_name saveto = os.path.join(to_dir, zip_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() downloader(url, saveto) return os.path.realpath(saveto) def _build_install_args(options): """ Build the arguments to 'python setup.py install' on the setuptools package. Returns list of command line arguments. """ return ['--user'] if options.user_install else [] def _parse_args(): """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, help='install in user site package') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, help='alternative URL from where to download the setuptools package') parser.add_option( '--insecure', dest='downloader_factory', action='store_const', const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) parser.add_option( '--version', help="Specify which version to download", default=DEFAULT_VERSION, ) parser.add_option( '--to-dir', help="Directory to save (and re-use) package", default=DEFAULT_SAVE_DIR, ) options, args = parser.parse_args() # positional arguments are ignored return options def _download_args(options): """Return args for download_setuptools function from cmdline args.""" return dict( version=options.version, download_base=options.download_base, downloader_factory=options.downloader_factory, to_dir=options.to_dir, ) def main(): """Install or upgrade setuptools and EasyInstall.""" options = _parse_args() archive = download_setuptools(**_download_args(options)) return _install(archive, _build_install_args(options)) if __name__ == '__main__': sys.exit(main()) pymongo-3.6.1/README.rst0000644000076600000240000001452013245621354015131 0ustar shanestaff00000000000000======= PyMongo ======= :Info: See `the mongo site `_ for more information. See `github `_ for the latest source. :Author: Mike Dirolf :Maintainer: Bernie Hackett About ===== The PyMongo distribution contains tools for interacting with MongoDB database from Python. The ``bson`` package is an implementation of the `BSON format `_ for Python. The ``pymongo`` package is a native Python driver for MongoDB. The ``gridfs`` package is a `gridfs `_ implementation on top of ``pymongo``. PyMongo supports MongoDB 2.6, 3.0, 3.2, 3.4, and 3.6. Support / Feedback ================== For issues with, questions about, or feedback for PyMongo, please look into our `support channels `_. Please do not email any of the PyMongo developers directly with issues or questions - you're more likely to get an answer on the `mongodb-user `_ list on Google Groups. Bugs / Feature Requests ======================= Think you’ve found a bug? Want to see a new feature in PyMongo? Please open a case in our issue management tool, JIRA: - `Create an account and login `_. - Navigate to `the PYTHON project `_. - Click **Create Issue** - Please provide as much information as possible about the issue type and how to reproduce it. Bug reports in JIRA for all driver projects (i.e. PYTHON, CSHARP, JAVA) and the Core Server (i.e. SERVER) project are **public**. How To Ask For Help ------------------- Please include all of the following information when opening an issue: - Detailed steps to reproduce the problem, including full traceback, if possible. - The exact python version used, with patch level:: $ python -c "import sys; print(sys.version)" - The exact version of PyMongo used, with patch level:: $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())" - The operating system and version (e.g. Windows 7, OSX 10.8, ...) - Web framework or asynchronous network library used, if any, with version (e.g. Django 1.7, mod_wsgi 4.3.0, gevent 1.0.1, Tornado 4.0.2, ...) Security Vulnerabilities ------------------------ If you’ve identified a security vulnerability in a driver or any other MongoDB project, please report it according to the `instructions here `_. Installation ============ PyMongo can be installed with `pip `_:: $ python -m pip install pymongo Or ``easy_install`` from `setuptools `_:: $ python -m easy_install pymongo You can also download the project source and do:: $ python setup.py install Do **not** install the "bson" package from pypi. PyMongo comes with its own bson package; doing "easy_install bson" installs a third-party package that is incompatible with PyMongo. Dependencies ============ PyMongo supports CPython 2.6, 2.7, 3.4+, PyPy, and PyPy3. Optional dependencies: GSSAPI authentication requires `pykerberos `_ on Unix or `WinKerberos `_ on Windows. The correct dependency can be installed automatically along with PyMongo:: $ python -m pip install pymongo[gssapi] Support for mongodb+srv:// URIs requires `dnspython `_:: $ python -m pip install pymongo[srv] TLS / SSL support may require `ipaddress `_ and `certifi `_ or `wincertstore `_ depending on the Python version in use. The necessary dependencies can be installed along with PyMongo:: $ python -m pip install pymongo[tls] You can install all dependencies automatically with the following command:: $ python -m pip install pymongo[gssapi,srv,tls] Other optional packages: - `backports.pbkdf2 `_, improves authentication performance with SCRAM-SHA-1, the default authentication mechanism for MongoDB 3.0+. It especially improves performance on Python versions older than 2.7.8. - `monotonic `_ adds support for a monotonic clock, which improves reliability in environments where clock adjustments are frequent. Not needed in Python 3. Additional dependencies are: - (to generate documentation) sphinx_ - (to run the tests under Python 2.6) unittest2_ Examples ======== Here's a basic example (for more see the *examples* section of the docs): .. code-block:: python >>> import pymongo >>> client = pymongo.MongoClient("localhost", 27017) >>> db = client.test >>> db.name u'test' >>> db.my_collection Collection(Database(MongoClient('localhost', 27017), u'test'), u'my_collection') >>> db.my_collection.insert_one({"x": 10}).inserted_id ObjectId('4aba15ebe23f6b53b0000000') >>> db.my_collection.insert_one({"x": 8}).inserted_id ObjectId('4aba160ee23f6b543e000000') >>> db.my_collection.insert_one({"x": 11}).inserted_id ObjectId('4aba160ee23f6b543e000002') >>> db.my_collection.find_one() {u'x': 10, u'_id': ObjectId('4aba15ebe23f6b53b0000000')} >>> for item in db.my_collection.find(): ... print(item["x"]) ... 10 8 11 >>> db.my_collection.create_index("x") u'x_1' >>> for item in db.my_collection.find().sort("x", pymongo.ASCENDING): ... print(item["x"]) ... 8 10 11 >>> [item["x"] for item in db.my_collection.find().limit(2).skip(1)] [8, 11] Documentation ============= You will need sphinx_ installed to generate the documentation. Documentation can be generated by running **python setup.py doc**. Generated documentation can be found in the *doc/build/html/* directory. Testing ======= The easiest way to run the tests is to run **python setup.py test** in the root of the distribution. Note that you will need unittest2_ to run the tests under Python 2.6. To verify that PyMongo works with Gevent's monkey-patching:: $ python green_framework_test.py gevent Or with Eventlet's:: $ python green_framework_test.py eventlet .. _sphinx: http://sphinx.pocoo.org/ .. _unittest2: https://pypi.python.org/pypi/unittest2